246 Commits

Author SHA1 Message Date
super
bca1335dc6 后端优化,增加新需求 2026-07-21 19:53:47 +08:00
super
ba48bbc6af 更新ASIN库新需求 2026-07-20 20:12:38 +08:00
super
cbde22ec79 视频任务管理更新 2026-07-19 22:23:26 +08:00
super
69784b8d32 添加新需求 2026-07-16 13:12:46 +08:00
super
8543dad514 修复后端BUG 2026-07-14 22:11:19 +08:00
super
8a9126e626 提交昨晚更新改动 2026-07-14 12:50:17 +08:00
super
d8ca8de537 完成视频部分的前后端 2026-07-11 19:42:42 +08:00
super
aab4ce942c 更新带货视频工作流接口和页面 2026-07-09 00:27:49 +08:00
super
0ecf5cd45a 后端优化修复问题 2026-06-18 18:53:16 +08:00
super
021b0c618b 修复图片下载问题 2026-06-16 22:19:55 +08:00
super
f44110fae5 新增数字人版本管理 2026-06-14 19:00:55 +08:00
super
ac19637ad6 处理修复BUG 2026-06-14 16:48:20 +08:00
super
8672668c33 处理下载为公共模块 2026-06-12 13:40:29 +08:00
super
17236265c6 修复删除asinBUG 2026-06-11 20:37:05 +08:00
super
c9efdf8b0e 修复菜单消失 2026-06-10 17:52:58 +08:00
super
c3d43aa26a 更新新需求-采集 2026-06-09 21:07:50 +08:00
super
b0462df3c1 更新新模块内容 更新新需求 2026-06-09 21:07:28 +08:00
super
967eedcad7 提交更新 2026-06-08 18:31:55 +08:00
super
98e0e54309 删除不必要模块 2026-06-08 11:51:08 +08:00
super
eeb27e21ed 删除本地APP项目 2026-06-08 11:49:46 +08:00
super
c85e5b278f 新增内容,新增拉起软件层 2026-06-08 09:19:15 +08:00
super
8ab647c419 提交更改 2026-06-05 21:25:03 +08:00
super
95ccb00d25 fix: reload similar asin chunks after coze flush 2026-06-05 21:18:26 +08:00
super
ee9cf5f8c6 fix brand result submission lock wait 2026-06-05 19:30:20 +08:00
super
e98a1a1207 feat: update task routing and coze handling 2026-06-03 17:14:50 +08:00
super
ea37d82d73 合并分支 2026-05-31 09:28:48 +08:00
super
9835831415 更新地址 更新后端 2026-05-29 22:53:44 +08:00
super
080625567b 同步 Gitee 应用代码更新 2026-05-29 19:47:23 +08:00
super
266c0f17c1 完成这个地址替换 2026-05-29 19:35:55 +08:00
super
225d13fb6e 提交优化更新 2026-05-29 16:49:18 +08:00
super
2ed1250604 增加公共下载进度、增加接收SKU、密钥分别存放 2026-05-28 16:41:20 +08:00
super
ca4a2cd07a python端登录完成 2026-05-27 12:41:59 +08:00
super
73ac9187a6 提交货源流程优化 2026-05-25 22:16:50 +08:00
super
7503e3fa8b 提交货源图片和相似ASIN优化 2026-05-24 21:59:21 +08:00
super
532438faba 提交最新的优化结果 2026-05-24 12:42:53 +08:00
super
a1376b51b0 优化图片尺寸 2026-05-24 02:31:58 +08:00
super
1e087c1aae 提交登录修改 2026-05-23 18:32:45 +08:00
super
2e2de02476 Merge gitee/master into local master (sync from old repo)
# Conflicts:
#	app/web_source/admin.html
#	app/web_source/brand-旧.html
#	app/web_source/home.html
#	app/web_source/index.html
#	app/web_source/login.html
#	backend/web_source/login.html
#	web_source/templates_backup/login.html
2026-05-23 12:00:12 +08:00
super
aeb4e1710c chore: 从 gitee 旧仓库恢复资源和软件更新包 2026-05-23 11:56:36 +08:00
super
b4b80cb572 更新这个货源 2026-05-22 09:41:31 +08:00
super
70e902e7ea 更新这个货源 2026-05-22 09:41:31 +08:00
super
14cebf22c3 提交一些更改 2026-05-20 09:05:28 +08:00
super
73d25fcbcb 提交一些更改 2026-05-20 09:05:28 +08:00
super
01ecde45c5 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-05-15 16:06:15 +08:00
super
a2ebfc0b81 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-05-15 16:06:15 +08:00
super
1dc2cac19a 修改完善这个专利部分 2026-05-15 16:06:12 +08:00
super
14db1e0cb1 修改完善这个专利部分 2026-05-15 16:06:12 +08:00
铭坤
01f4dbb83a modified: amazon/similar_asin.py 2026-05-15 11:33:02 +08:00
铭坤
0d63d6d75a modified: amazon/similar_asin.py 2026-05-15 11:33:02 +08:00
铭坤
fe692fcd1d modified: amazon/chrome_base.py
modified:   amazon/detail_spider.py
	modified:   amazon/match_action.py
	modified:   amazon/similar_asin.py
	deleted:    amazon/user_data/chrome_data/BrowserMetrics-spare.pma
	deleted:    amazon/user_data/chrome_data/BrowserMetrics/BrowserMetrics-69FAEF07-558.pma
	deleted:    amazon/user_data/chrome_data/Crashpad/metadata
	deleted:    amazon/user_data/chrome_data/Crashpad/settings.dat
2026-05-13 16:51:29 +08:00
铭坤
79b177b345 modified: amazon/chrome_base.py
modified:   amazon/detail_spider.py
	modified:   amazon/match_action.py
	modified:   amazon/similar_asin.py
	deleted:    amazon/user_data/chrome_data/BrowserMetrics-spare.pma
	deleted:    amazon/user_data/chrome_data/BrowserMetrics/BrowserMetrics-69FAEF07-558.pma
	deleted:    amazon/user_data/chrome_data/Crashpad/metadata
	deleted:    amazon/user_data/chrome_data/Crashpad/settings.dat
2026-05-13 16:51:29 +08:00
super
18b0c2d211 软件前端更新 2026-05-12 23:25:12 +08:00
super
e4e01c1686 软件前端更新 2026-05-12 23:25:12 +08:00
super
648e7d2f14 后台管理接口修复 后端BUG修复 2026-05-12 21:03:42 +08:00
super
0d78d63437 后台管理接口修复 后端BUG修复 2026-05-12 21:03:42 +08:00
super
289b1c67ce 提交货源采集更新 2026-05-10 01:00:08 +08:00
super
5c990e651e 提交货源采集更新 2026-05-10 01:00:08 +08:00
super
52095eb992 更新处理相关内容 2026-05-09 00:20:52 +08:00
super
a4e4745921 更新处理相关内容 2026-05-09 00:20:52 +08:00
super
a2d0bd3c1c Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-05-07 16:22:42 +08:00
super
683a3934fc 提交打包 2026-05-07 16:21:35 +08:00
铭坤
b0514c3e9a modified: amazon/price_match.py
modified:   config.py
2026-05-07 11:14:15 +08:00
铭坤
8be12ca9bc new file: requirements.txt 2026-05-07 11:00:44 +08:00
super
afeeb8327c 更新 2026-05-06 23:00:57 +08:00
super
4d2da94692 同步更新 2026-05-06 23:00:39 +08:00
super
b746812e23 更新外观模块 2026-05-06 22:41:26 +08:00
铭坤
3fbbf87c10 modified: .env
modified:   amazon/approve.py
	modified:   amazon/chrome_base.py
	modified:   amazon/detail_spider.py
	modified:   amazon/main.py
	modified:   amazon/price_match.py
	modified:   amazon/similar_asin.py
	modified:   amazon/tool.py
	new file:   amazon/user_data/chrome_data/BrowserMetrics-spare.pma
	new file:   amazon/user_data/chrome_data/BrowserMetrics/BrowserMetrics-69FAEF07-558.pma
2026-05-06 22:30:44 +08:00
super
c3810fdb7b 提交更新 2026-05-06 00:01:49 +08:00
super
b16c63225f 处理一些合并冲突 2026-05-05 14:11:35 +08:00
铭坤
45a45dc0fe Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin
modified:   amazon/detail_spider.py
	modified:   amazon/main.py
2026-05-04 19:17:11 +08:00
铭坤
c315bb350c new file: amazon/amazon_base.py
modified:   amazon/approve.py
	modified:   amazon/asin_status.py
	new file:   amazon/chrome_base.py
	modified:   amazon/del_brand.py
	modified:   amazon/detail_spider.py
	new file:   "amazon/detail_spider_\345\244\207\344\273\275.py"
	modified:   amazon/main.py
	modified:   amazon/match_action.py
	modified:   amazon/price_match.py
	new file:   amazon/similar_asin.py
	modified:   app.py
	new file:   assets/appearance-patent-Br1dtmol.css
	new file:   assets/appearance-patent-D1DgeYhe.css
	new file:   assets/delete-brand-BfMLVSQU.css
	new file:   assets/patrol-delete-BAzbYtgc.css
	new file:   assets/price-track-Hozgt_em.css
	new file:   assets/product-risk-CbX7bwZi.css
	new file:   assets/query-asin-B4RsOiza.css
	new file:   assets/shop-match-B2HWAgBY.css
	modified:   main.py
2026-05-04 19:15:50 +08:00
super
6d398b66bc 异常记录检测修改 2026-05-04 19:04:55 +08:00
super
934919c699 稳定组成这个文件优化 2026-05-02 16:40:13 +08:00
super
12d84a22e4 更新 2026-05-02 09:31:14 +08:00
super
dc457aff9e Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-05-02 09:29:43 +08:00
super
578847d596 提交更新 2026-05-02 09:28:26 +08:00
koko
6416c14f20 chore: serve bundled assets from app directory 2026-05-02 02:12:14 +08:00
koko
add6e6aeb5 chore: remove stale web assets 2026-05-02 02:07:13 +08:00
koko
7968cef8c3 fix: update patrol delete progress handling 2026-05-02 02:03:49 +08:00
koko
34148b5d8a Merge patrol delete tests and remove pycache artifacts 2026-05-01 22:46:11 +08:00
koko
79e408333d Merge branch 'dev/koko'
# Conflicts:
#	app/amazon/patrol_delete.py
2026-05-01 22:41:09 +08:00
koko
74a7cd22b0 Fix patrol delete condition handling 2026-05-01 22:36:37 +08:00
super
e1cca219b8 完善多店铺开启多窗口、外观进度条等 2026-05-01 21:09:54 +08:00
super
4419c5bacd 更新处理这个外观部分 2026-05-01 00:09:14 +08:00
铭坤
4eec0dd5a4 modified: amazon/__pycache__/price_match.cpython-39.pyc
modified:   amazon/price_match.py
	modified:   assets/appearance-patent.js
	modified:   assets/delete-brand.js
	modified:   assets/patrol-delete.js
	modified:   assets/price-track.js
	modified:   assets/product-risk.js
	modified:   assets/query-asin.js
	modified:   assets/shop-match.js
	modified:   new_web_source/delete-brand.html
2026-04-30 21:57:37 +08:00
koko
8e60f50616 Merge origin master into local master 2026-04-30 21:46:00 +08:00
koko
f85e3be40e Harden patrol delete logging 2026-04-30 21:43:15 +08:00
koko
7f9116f12a Merge patrol delete changes into master 2026-04-30 21:40:35 +08:00
super
11f0d2c745 修复商品风险多店铺问题 2026-04-30 21:31:11 +08:00
super
955a6439a0 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-30 11:38:13 +08:00
super
172917ac41 优化后台业务 2026-04-30 11:36:58 +08:00
铭坤
a5f6898d06 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin
app/amazon/__pycache__/approve.cpython-39.pyc
	app/amazon/__pycache__/asin_status.cpython-39.pyc
	app/amazon/__pycache__/del_brand.cpython-39.pyc
	app/amazon/__pycache__/match_action.cpython-39.pyc
	app/amazon/__pycache__/price_match.cpython-39.pyc
2026-04-30 11:00:02 +08:00
铭坤
83bc90ecf2 modified: amazon/__pycache__/approve.cpython-39.pyc
modified:   amazon/__pycache__/asin_status.cpython-39.pyc
	modified:   amazon/__pycache__/del_brand.cpython-39.pyc
	modified:   amazon/__pycache__/match_action.cpython-39.pyc
	modified:   amazon/__pycache__/price_match.cpython-39.pyc
	modified:   amazon/approve.py
	modified:   amazon/asin_status.py
	modified:   amazon/del_brand.py
	modified:   amazon/match_action.py
	modified:   amazon/price_match.py
	modified:   main.py
2026-04-30 10:52:52 +08:00
koko
cbfc29e953 Remove ts script 2026-04-28 22:41:56 +08:00
koko
b66465aa6a chore: update env 2026-04-28 22:40:39 +08:00
koko
fbd1a4dbea Remove README from master 2026-04-28 22:39:38 +08:00
koko
be8a8492a9 Remove tracked .rtk config 2026-04-28 22:38:17 +08:00
koko
b352eea21d Stop tracking remaining Python bytecode on master
Master still tracked a small set of generated __pycache__ artifacts after merging dev/koko. The ignore rules already match dev/koko, so this commit only removes the remaining bytecode files from the Git index while leaving local files on disk.

Constraint: dev/koko .gitignore already contains recursive __pycache__ ignore coverage

Rejected: Delete local cache directories | repository tracking cleanup only requires --cached removal

Confidence: high

Scope-risk: narrow

Tested: git ls-files shows no remaining paths containing __pycache__/

Not-tested: Application runtime; generated bytecode tracking only
2026-04-28 22:33:27 +08:00
koko
51bf0aaa8c Merge dev/koko into master preserving master conflict behavior
Conflict resolution kept master-side changes for non-patrol-delete files as requested. No app/amazon/patrol_delete.py conflict was present. Existing dev/koko assets were retained so /assets resources required by new_web_source pages resolve in the Flask app.

Constraint: User requested authentication-requiring git operations be reported as commands only

Rejected: Rework non-conflicting dev/koko changes during merge | scope was conflict resolution, not feature review

Confidence: high

Scope-risk: moderate

Tested: uv run python -c Flask test client returned 200 for /new_web_source/patrol-delete.html, /assets/patrol-delete.js, /assets/pywebview-C66x_2Dh.js

Tested: uv run python -m py_compile amazon\\patrol_delete.py amazon\\detail_spider.py amazon\\price_match.py main.py

Tested: uv run --group dev pytest tests\\test_patrol_delete.py tests\\test_amazon_base.py

Not-tested: authenticated Git push/pull; Java backend build
2026-04-28 22:31:38 +08:00
koko
28a752265c fix: brand html 2026-04-28 22:14:41 +08:00
koko
1d9d138423 fix: update frontend 2026-04-28 21:42:30 +08:00
铭坤
8c5edc6ad6 new file: assets/_plugin-vue_export-helper-Dh4yMZWw.js
new file:   assets/_plugin-vue_export-helper-xvHHTGU_.css
	new file:   assets/appearance-patent-BNTrbLJh.css
	new file:   assets/appearance-patent-BPCrG75P.css
	new file:   assets/appearance-patent.js
	new file:   assets/brand-1nUVsWrm.js
	new file:   assets/brand-5-W0Wonr.js
	new file:   assets/brand-BATjYfEL.js
	new file:   assets/brand-BZije8D7.js
	new file:   assets/brand-C0Q5LZ0P.js
	new file:   assets/brand-CFZZtTv8.js
	new file:   assets/brand-CKgrwtni.js
	new file:   assets/brand-CRmRvyGD.js
	new file:   assets/brand-CXHemZwJ.js
	new file:   assets/brand-DBpLCgAF.js
	new file:   assets/brand-DRn3N4FA.js
	new file:   assets/brand-DZ5_PGnw.js
	new file:   assets/brand-DysFlQbZ.js
	new file:   assets/brand-UU-ckLq6.js
	new file:   assets/convert-1dkFPbKa.css
	new file:   assets/convert-6b0yq-M0.css
	new file:   assets/convert-7wWJ02Tw.css
	new file:   assets/convert-BKSNvX8i.css
	new file:   assets/convert-BxkpMHO9.css
	new file:   assets/convert-CQ22EBpm.css
	new file:   assets/convert-Cfr3mUjs.css
	new file:   assets/convert-D4Yv2oRy.css
	new file:   assets/convert-DwbQug2y.css
	new file:   assets/dedupe-B0eqpVpj.css
	new file:   assets/dedupe-BHE_BCAJ.css
	new file:   assets/dedupe-BpNHwt51.css
	new file:   assets/dedupe-CS18ls2z.css
	new file:   assets/dedupe-DNlVfFj-.css
	new file:   assets/dedupe-Day_nGxq.css
	new file:   assets/dedupe-DyzjKbsT.css
	new file:   assets/dedupe-DzFvTfoj.css
	new file:   assets/dedupe-rhGu1E8E.css
	new file:   assets/delete-brand-BNvh4ru4.css
	new file:   assets/delete-brand-BP3XWKAC.css
	new file:   assets/delete-brand-BRIDBaoM.css
	new file:   assets/delete-brand-BcZomrN3.css
	new file:   assets/delete-brand-BgPrpPsj.css
	new file:   assets/delete-brand-ByeMdpEk.css
	new file:   assets/delete-brand-C-Rj3NMH.css
	new file:   assets/delete-brand-CDZTH98q.css
	new file:   assets/delete-brand-CS44Bk-y.css
	new file:   assets/delete-brand-CWLpe7lu.css
	new file:   assets/delete-brand-CdxGWtvK.css
	new file:   assets/delete-brand-Chu8awlt.css
	new file:   assets/delete-brand-CndjrFFS.css
	new file:   assets/delete-brand-CpNTe9WM.css
	new file:   assets/delete-brand-D7t16Ukk.css
	new file:   assets/delete-brand-DE0hLEak.css
	new file:   assets/delete-brand-DLyg-H8J.css
	new file:   assets/delete-brand-DO6pdlC3.css
	new file:   assets/delete-brand-DXSeQxXc.css
	new file:   assets/delete-brand-DZ6zYt8o.css
	new file:   assets/delete-brand-DfRLfrtm.css
	new file:   assets/delete-brand-Dl7T0pmm.css
	new file:   assets/delete-brand-Dt_MS9as.css
	new file:   assets/delete-brand-DvND7vBL.css
	new file:   assets/delete-brand-_jGFWTAn.css
	new file:   assets/el-alert-Ct0RpqUD.css
	new file:   assets/el-input-CezJelDw.css
	new file:   assets/el-input-xaztPnzw.css
	new file:   assets/el-table-column-Cy4YJvw0.css
	new file:   assets/listingFilters-B6-M_FNM.js
	new file:   assets/listingFilters-BmlAzw7M.css
	new file:   assets/listingFilters-BpGOU_pJ.js
	new file:   assets/listingFilters-CK58rX4v.css
	new file:   assets/patrol-delete-BPu1AAPM.css
	new file:   assets/patrol-delete-BctopIU4.css
	new file:   assets/patrol-delete-CSekhzSS.css
	new file:   assets/patrol-delete-DIvFZUTh.css
	new file:   assets/patrol-delete.js
	new file:   assets/price-track-1SjcUfP4.css
	new file:   assets/price-track-85NbckiJ.css
	new file:   assets/price-track-B5FVfJb_.css
	new file:   assets/price-track-BO5PtL0W.css
	new file:   assets/price-track-BccJT4Qr.css
	new file:   assets/price-track-BlSD-0un.css
	new file:   assets/price-track-BlVVl2eV.css
	new file:   assets/price-track-BwgEuRxe.css
	new file:   assets/price-track-CWQD_A7D.css
	new file:   assets/price-track-DNTcdRH8.css
	new file:   assets/price-track-Dv4EZPhf.css
	new file:   assets/price-track-Ji5JAoeR.css
	new file:   assets/price-track-RLJUgMl_.css
	new file:   assets/price-track-z8RU_FlW.css
	new file:   assets/price-track.js
	new file:   assets/product-risk-1G_ztqNY.css
	new file:   assets/product-risk-A5bhfaTH.css
	new file:   assets/product-risk-BBMcuGAw.css
	new file:   assets/product-risk-C3Yn2wAL.css
	new file:   assets/product-risk-CG-1Iq4P.css
	new file:   assets/product-risk-COeM91v5.css
	new file:   assets/product-risk-ClWs7d57.css
	new file:   assets/product-risk-CtpNfape.css
	new file:   assets/product-risk-DBsBE_mg.css
	new file:   assets/product-risk-DRYaWmI2.css
	new file:   assets/product-risk-Db49F4J5.css
	new file:   assets/product-risk-DoO8ZHM0.css
	new file:   assets/product-risk-DpSMKBCA.css
	new file:   assets/product-risk-DqFPFWEv.css
	new file:   assets/product-risk-cfL3Ch5-.css
	new file:   assets/product-risk-eXMv239F.css
	new file:   assets/product-risk-un2cxu--.css
	new file:   assets/product-risk.js
	new file:   assets/pywebview-3PePjYSu.js
	new file:   assets/pywebview-8G_9KMXd.css
	new file:   assets/pywebview-9YBa--7x.js
	new file:   assets/pywebview-B6ZNXusn.css
	new file:   assets/pywebview-BCPdlDdb.js
	new file:   assets/pywebview-BLoeiUcF.css
	new file:   assets/pywebview-BRrok0pS.js
	new file:   assets/pywebview-BT0f6KST.css
	new file:   assets/pywebview-BbIq-QPS.css
	new file:   assets/pywebview-Bc09DixN.js
	new file:   assets/pywebview-Bl6M97uH.js
	new file:   assets/pywebview-BnpJyyGZ.js
	new file:   assets/pywebview-Bt854mYs.js
	new file:   assets/pywebview-C5RGB8QW.css
	new file:   assets/pywebview-C66x_2Dh.js
	new file:   assets/pywebview-C8Bik-Sc.js
	new file:   assets/pywebview-CGwF8TuM.js
	new file:   assets/pywebview-CJLylbJu.js
	new file:   assets/pywebview-CeWJDVeG.js
	new file:   assets/pywebview-Chz98E9c.css
	new file:   assets/pywebview-ClWy2SbE.js
	new file:   assets/pywebview-Cq_E2BnJ.js
	new file:   assets/pywebview-Cs1Kot1q.js
	new file:   assets/pywebview-D-PKWjUg.js
	new file:   assets/pywebview-D-mMH8F6.css
	new file:   assets/pywebview-D4gpiFjY.js
	new file:   assets/pywebview-D74jYk6P.css
	new file:   assets/pywebview-D7_PdvyM.js
	new file:   assets/pywebview-D808cNhy.js
	new file:   assets/pywebview-DVgP2-kW.js
	new file:   assets/pywebview-Dee3nQjE.js
	new file:   assets/pywebview-DiP0HdY6.js
	new file:   assets/pywebview-Dp5dN8OO.css
	new file:   assets/pywebview-DuyK2jB1.js
	new file:   assets/pywebview-IqgMfeBe.css
	new file:   assets/pywebview-MkjZQlBu.css
	new file:   assets/pywebview-Q4glvYu5.css
	new file:   assets/pywebview-Rm5Hhqpf.js
	new file:   assets/pywebview-V_UIajJm.js
	new file:   assets/pywebview-hj_EXwAP.css
	new file:   assets/pywebview-ij4pgMq8.css
	new file:   assets/pywebview-leZCedEt.js
	new file:   assets/query-asin-B0yMJ9Or.css
	new file:   assets/query-asin-B5e5m-vs.css
	new file:   assets/query-asin-Bv7Jtgqa.css
	new file:   assets/query-asin-DZwJfwHU.css
	new file:   assets/query-asin.js
	new file:   assets/shop-match-Banz7DWL.css
	new file:   assets/shop-match-BiZP8IGM.css
	new file:   assets/shop-match-BrrGkYV7.css
	new file:   assets/shop-match-CVFsXUvd.css
	new file:   assets/shop-match-CfGJEHz0.css
	new file:   assets/shop-match-CfGWwDtw.css
	new file:   assets/shop-match-D0DGETCs.css
	new file:   assets/shop-match-DFEKOpvV.css
	new file:   assets/shop-match-DNFA8ceR.css
	new file:   assets/shop-match-DhT1CvMP.css
	new file:   assets/shop-match-L0xHTaQ2.css
	new file:   assets/shop-match-MsxXMG_G.css
	new file:   assets/shop-match-g3eRKhTP.css
	new file:   assets/shop-match.js
	new file:   assets/split-BoBVrLdC.css
	new file:   assets/split-BsJfbBqn.css
	new file:   assets/split-CRUIYKS6.css
	new file:   assets/split-DAa_Usoh.css
	new file:   assets/split-DidKJ84l.css
	new file:   assets/split-RNZYJ-zZ.css
	new file:   assets/split-R_8V5xgh.css
	new file:   assets/split-_O6NHqDj.css
	new file:   assets/split-mwLUJT1K.css
	new file:   assets/zh-cn-BjWx61Lu.js
	new file:   assets/zh-cn-C0iS6aoJ.js
	new file:   assets/zh-cn-CuE9Zzgz.js
	new file:   assets/zh-cn-DyKmB71i.js
	new file:   assets/zh-cn-Z9PBnl-n.js
2026-04-28 21:36:49 +08:00
铭坤
ee6fb5d33a modified: amazon/__pycache__/detail_spider.cpython-39.pyc
modified:   amazon/__pycache__/price_match.cpython-39.pyc
	modified:   amazon/detail_spider.py
	modified:   amazon/price_match.py
	modified:   blueprints/__pycache__/admin.cpython-39.pyc
	modified:   blueprints/__pycache__/brand.cpython-39.pyc
	modified:   blueprints/__pycache__/main.cpython-39.pyc
	modified:   brand_spider/__pycache__/main.cpython-39.pyc
	modified:   main.py
	modified:   web_source/brand.html
2026-04-28 21:11:48 +08:00
koko
cd2b437be5 Stop tracking generated Python bytecode
Generated __pycache__ files were already ignored in some paths but remained tracked in the repository, which caused merge noise and binary conflicts. This removes the tracked bytecode from the index and adds an explicit recursive ignore rule so future Python runs do not reintroduce them.

Constraint: Existing repository history already tracked pycache files across app, backend, and source_code

Rejected: Delete local pycache directories from disk | only repository tracking needed to be removed

Confidence: high

Scope-risk: narrow

Directive: Do not force-add __pycache__ or *.pyc files unless there is a documented runtime packaging reason

Tested: git ls-files shows no remaining paths containing __pycache__/

Not-tested: Application runtime; change only affects tracked generated artifacts and ignore rules
2026-04-28 20:57:53 +08:00
koko
a504e7a13b Merge master into dev/koko after patrol delete fixes
The merge keeps dev/koko patrol-delete asset fallback behavior while accepting the latest master feature work. Conflicts were limited to ignore rules, local environment/config artifacts, tracked pycache binaries, and the Flask main blueprint.

Constraint: master and dev/koko both edited app/blueprints/main.py around brand and asset serving

Rejected: Drop dev/koko asset fallback logic | patrol delete and rebuilt Vite assets still need hashed asset resolution across app/new_web_source locations

Confidence: medium

Scope-risk: broad

Directive: app/.env remains a tracked local-config file in this repository; do not print or normalize secrets during conflict handling

Tested: uv run python -m py_compile blueprints\\main.py amazon\\base.py amazon\\main.py

Tested: uv run --group dev pytest tests\\test_amazon_base.py tests\\test_patrol_delete.py

Not-tested: Full backend-java/frontend-vue test suites
2026-04-28 20:55:27 +08:00
koko
325687d532 Stabilize patrol delete automation against delayed Amazon UI
The patrol-delete flow now waits for country and listing-status controls before acting, uses the dedicated PatrolDeleteTask entry point, and includes the browser helper scripts required by the bulk-delete path. Tests cover delayed dropdown readiness, payload rejection/retry behavior, complete-draft normalization, filtered bulk deletion, and asset fallback lookup.

Constraint: Amazon listing UI exposes status controls through dynamic KAT components and delayed option rendering

Rejected: Keep row-by-row delete flow | bulk selection is the current implemented path and is covered by the added helper scripts

Confidence: high

Scope-risk: moderate

Directive: Do not remove the patrol_delete JS helper files without checking app/amazon/patrol_delete.py script loading

Tested: uv run --group dev pytest tests\\test_amazon_base.py tests\\test_patrol_delete.py

Tested: uv run python -m py_compile amazon\\base.py amazon\\main.py blueprints\\main.py

Not-tested: Real Amazon Seller Central browser session
2026-04-28 20:46:50 +08:00
super
9760d1171c 提交跟新 2026-04-28 17:16:04 +08:00
super
d42ed57119 提交暂存 2026-04-28 16:57:05 +08:00
super
18208f691c 更新内容 2026-04-28 16:56:53 +08:00
super
c06a4d0e06 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-28 15:33:42 +08:00
super
f21934e55b 合并错误信息 2026-04-28 15:25:34 +08:00
铭坤
515135c120 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-28 15:23:55 +08:00
铭坤
42bdd59c71 modified: .env
modified:   amazon/__pycache__/approve.cpython-39.pyc
	modified:   amazon/__pycache__/detail_spider.cpython-39.pyc
	modified:   amazon/__pycache__/price_match.cpython-39.pyc
	modified:   amazon/approve.py
	modified:   amazon/detail_spider.py
	modified:   amazon/price_match.py
	modified:   assets/convert.js
	modified:   assets/dedupe.js
	modified:   assets/delete-brand.js
	modified:   assets/split.js
	modified:   blueprints/__pycache__/admin.cpython-39.pyc
	modified:   blueprints/__pycache__/brand.cpython-39.pyc
	modified:   blueprints/__pycache__/main.cpython-39.pyc
	modified:   blueprints/main.py
	modified:   brand_spider/__pycache__/main.cpython-39.pyc
	modified:   new_web_source/convert.html
	modified:   new_web_source/dedupe.html
	modified:   new_web_source/delete-brand.html
	modified:   new_web_source/split.html
	modified:   tool/__pycache__/devices.cpython-39.pyc
2026-04-28 15:21:57 +08:00
koko
8be85fd332 Expose patrol delete progress in logs
Patrol delete can spend a long time switching countries, reading listing states, and confirming deletes, so the flow now emits concise progress logs around each high-value step. The logs avoid full DOM dumps outside existing diagnostics and focus on task, shop, country, status, deletion counts, and callback boundaries.

Constraint: Existing patrol delete runs are browser-driven and need observable progress without changing deletion behavior
Rejected: Log every DOM probe | too noisy for routine task monitoring
Confidence: high
Scope-risk: narrow
Tested: uv run --group dev pytest tests\\test_patrol_delete.py
Tested: uv run --group dev ruff check amazon\\patrol_delete.py
Tested: uv run python -m py_compile amazon\\patrol_delete.py
Not-tested: Live Seller Central browser run
2026-04-28 01:23:58 +08:00
super
e4bf104ae2 架构更改完成 2026-04-27 18:45:56 +08:00
super
5105bf7049 增加rufts 2026-04-27 18:45:33 +08:00
koko
08997e9e20 Rename patrol delete tests
Keep the patrol deletion test module aligned with the renamed patrol_delete implementation and update test function names so future searches no longer point at the old product module name.

Constraint: This is a naming-only follow-up to the patrol_delete module rename

Confidence: high

Scope-risk: narrow

Tested: uv run pytest tests/test_patrol_delete.py
2026-04-27 17:59:46 +08:00
koko
9146d17625 Align patrol delete module naming
Rename the patrol deletion implementation and browser helper scripts away from the generic product name so the filesystem matches the queued patrol-delete task domain. Update imports, script loading, and tests to use the patrol_delete module path.

Constraint: Existing task class remains ProductTask to avoid broad API churn beyond the requested file and script naming

Rejected: Rename ProductTask class now | would expand the change into a larger API update across tests and task dispatch

Confidence: high

Scope-risk: narrow

Directive: Keep patrol-delete script paths under app/amazon/scripts/patrol_delete when adding new helper scripts

Tested: uv run pytest tests/test_product.py

Not-tested: Remote push before this commit due previous SSH publickey failure
2026-04-27 17:58:40 +08:00
koko
1aca3d6152 Bring Amazon collection tasks into dev/koko
Merge origin/master into dev/koko while preserving the patrol delete task added on dev/koko. The task dispatcher now recognizes both patrol deletion and appearance patent collection, and the master-side SpiderTask implementation is included.

Constraint: origin fetch and push over SSH are blocked in this environment by Gitee publickey authentication

Rejected: Overwrite dev/koko with master | would drop the patrol delete branch work

Confidence: medium

Scope-risk: moderate

Directive: Do not remove either patrol-delete-run or appearance-patent-run without checking queued task producers

Tested: python -m py_compile app/amazon/approve.py app/amazon/asin_status.py app/amazon/detail_spider.py app/amazon/main.py app/amazon/match_action.py app/amazon/price_match.py

Not-tested: Remote push, blocked by SSH publickey authentication
2026-04-27 17:36:30 +08:00
super
052bb31aea Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-27 09:20:04 +08:00
super
225b525ba1 后端架构更新 2026-04-27 09:18:10 +08:00
koko
e1543416e3 feat: patrol delete 2026-04-27 00:23:44 +08:00
铭坤
0b57aea5f6 modified: amazon/__pycache__/approve.cpython-39.pyc
modified:   amazon/__pycache__/asin_status.cpython-39.pyc
	new file:   amazon/__pycache__/detail_spider.cpython-39.pyc
	modified:   amazon/__pycache__/main.cpython-39.pyc
	modified:   amazon/__pycache__/match_action.cpython-39.pyc
	modified:   amazon/__pycache__/price_match.cpython-39.pyc
	modified:   amazon/approve.py
	modified:   amazon/asin_status.py
	new file:   amazon/detail_spider.py
	modified:   amazon/main.py
	modified:   amazon/match_action.py
	modified:   amazon/price_match.py
	modified:   assets/convert.js
	modified:   assets/dedupe.js
	modified:   assets/delete-brand.js
	modified:   assets/split.js
	modified:   new_web_source/convert.html
	modified:   new_web_source/dedupe.html
	modified:   new_web_source/delete-brand.html
	modified:   new_web_source/split.html
	modified:   web_source/brand.html
2026-04-27 00:21:37 +08:00
koko
a04c2f9a19 feat: update amazon driver 2026-04-25 17:59:12 +08:00
koko
c352c34501 feat: update amazon driver 2026-04-25 17:58:28 +08:00
koko
5c9671df44 feat: update driver BaseClass 2026-04-25 17:14:23 +08:00
koko
8a8d3c5cd9 Merge branch 'master' into dev/koko 2026-04-25 15:47:02 +08:00
铭坤
d272afae1c modified: amazon/__pycache__/approve.cpython-39.pyc
new file:   amazon/__pycache__/asin_status.cpython-39.pyc
	modified:   amazon/__pycache__/main.cpython-39.pyc
	modified:   amazon/__pycache__/match_action.cpython-39.pyc
	modified:   amazon/__pycache__/price_match.cpython-39.pyc
	modified:   amazon/approve.py
	modified:   amazon/asin_status.py
	modified:   amazon/main.py
	modified:   amazon/match_action.py
	modified:   amazon/price_match.py
	modified:   web_source/brand.html
	modified:   web_source/templates_backup/brand.html
2026-04-24 23:00:59 +08:00
super
03e697f5d3 后台查询Asin文档更新 2026-04-24 15:14:14 +08:00
super
c502afb588 完成查询asin开发 2026-04-23 20:48:57 +08:00
koko
3fada5d198 Merge branch 'master' into dev/koko 2026-04-23 11:49:50 +00:00
super
c0fdea6570 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-23 15:25:49 +08:00
super
0391cb223f 完成后端架构重构等 2026-04-23 15:25:41 +08:00
koko
b25111e4b4 Merge remote-tracking branch 'origin/master' into dev/koko 2026-04-23 03:16:15 +00:00
铭坤
2f2db4986e modified: app/amazon/__pycache__/approve.cpython-39.pyc
modified:   app/amazon/__pycache__/del_brand.cpython-39.pyc
	modified:   app/amazon/__pycache__/match_action.cpython-39.pyc
	modified:   app/amazon/__pycache__/price_match.cpython-39.pyc
	modified:   app/amazon/approve.py
	modified:   app/amazon/del_brand.py
	modified:   app/amazon/match_action.py
	modified:   app/amazon/price_match.py
	modified:   app/assets/convert.js
	modified:   app/assets/dedupe.js
	modified:   app/assets/delete-brand.js
	modified:   app/assets/split.js
	modified:   app/new_web_source/convert.html
	modified:   app/new_web_source/dedupe.html
	modified:   app/new_web_source/delete-brand.html
	modified:   app/new_web_source/split.html
2026-04-23 10:41:07 +08:00
koko
6f970b3783 Clarify the current automation example in onboarding docs
Update the README so new contributors start from the latest patrol-delete automation flow instead of treating delete-brand as the primary example. Keep the older delete-brand path as contrast for the file-driven workflow.

Constraint: Recent repository changes made patrol-delete the newest automation module, so the onboarding example needed to match current development reality
Rejected: Leave delete-brand as the primary example | would mislead readers about the latest automation entry point
Confidence: high
Scope-risk: narrow
Reversibility: clean
Directive: Revisit this section when a newer automation module replaces patrol-delete as the primary example
Tested: README diff review
Not-tested: lint, typecheck, unit/integration tests not run (documentation-only change)
2026-04-22 08:33:02 +00:00
铭坤
95d5c82474 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-22 15:58:46 +08:00
铭坤
87507708ce new file: admin.html
new file:   "brand - \345\211\257\346\234\254.html"
	new file:   "brand-\346\227\247.html"
	new file:   brand.html
	new file:   home.html
	new file:   index.html
	new file:   login.html
2026-04-22 15:57:56 +08:00
koko
c6ae7ca170 Document repo structure and ignore local OMX state
Add a root README that explains the active runtime layers and onboarding path, and keep local OMX session state out of version control so developer tooling does not interfere with pulls.

Constraint: Local OMX state is developer-specific and should not block branch sync or appear in shared history
Rejected: Keep onboarding notes untracked | easy to lose and hard to share with the team
Confidence: high
Scope-risk: narrow
Reversibility: clean
Directive: Keep the README aligned with the active app/backend-java/frontend-vue execution path; revisit if the runtime architecture changes
Tested: git diff review; git status after staging
Not-tested: lint, typecheck, unit/integration tests not run (docs and ignore rules only)
2026-04-22 07:16:04 +00:00
super
6894f9cc57 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-22 01:09:51 +08:00
super
524d8763ce 处理后台跳转错误 2026-04-22 01:09:47 +08:00
super
1e845a1510 处理后台管理系统、修复BUG、处理权限 2026-04-22 01:09:28 +08:00
铭坤
0341838d19 modified: amazon/__pycache__/approve.cpython-39.pyc
modified:   amazon/__pycache__/main.cpython-39.pyc
	modified:   amazon/__pycache__/match_action.cpython-39.pyc
	new file:   amazon/__pycache__/price_match.cpython-39.pyc
	modified:   amazon/__pycache__/tool.cpython-39.pyc
	modified:   amazon/approve.py
	new file:   amazon/asin_status.py
	modified:   amazon/main.py
	modified:   amazon/price_match.py
	new file:   "amazon/price_match_\346\227\247.py"
	modified:   amazon/tool.py
	modified:   assets/convert.js
	modified:   assets/dedupe.js
	modified:   assets/delete-brand.js
	modified:   assets/split.js
	modified:   new_web_source/convert.html
	modified:   new_web_source/dedupe.html
	modified:   new_web_source/delete-brand.html
	modified:   new_web_source/split.html
	deleted:    web_source/admin.html
	deleted:    "web_source/brand - \345\211\257\346\234\254.html"
	deleted:    "web_source/brand-\346\227\247.html"
	deleted:    web_source/brand.html
	deleted:    web_source/home.html
	deleted:    web_source/index.html
	deleted:    web_source/login.html
2026-04-22 00:51:41 +08:00
super
ea35273597 改造后端服务,处理后台管理新增组(部门概念) 2026-04-22 00:32:58 +08:00
super
72c8167472 更新跟价相关内容修改 2026-04-20 00:41:49 +08:00
super
9b1138c83e Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-19 15:47:37 +08:00
super
dc5e23892f 提交改造app权限 2026-04-19 15:47:20 +08:00
铭坤
4f8cbc3e38 new file: update/PyQt5/Qt/.keep_dir.txt
new file:   update/PyQt5/Qt5/.keep_dir.txt
	new file:   update/PyQt5/QtCore.pyd
	new file:   update/PyQt5/QtGui.pyd
	new file:   update/PyQt5/QtWidgets.pyd
	new file:   update/PyQt5/qt-plugins/iconengines/qsvgicon.dll
	new file:   update/PyQt5/qt-plugins/imageformats/qgif.dll
	new file:   update/PyQt5/qt-plugins/imageformats/qicns.dll
	new file:   update/PyQt5/qt-plugins/imageformats/qico.dll
	new file:   update/PyQt5/qt-plugins/imageformats/qjpeg.dll
	new file:   update/PyQt5/qt-plugins/imageformats/qsvg.dll
	new file:   update/PyQt5/qt-plugins/imageformats/qtga.dll
	new file:   update/PyQt5/qt-plugins/imageformats/qtiff.dll
	new file:   update/PyQt5/qt-plugins/imageformats/qwbmp.dll
	new file:   update/PyQt5/qt-plugins/imageformats/qwebp.dll
	new file:   update/PyQt5/qt-plugins/mediaservice/dsengine.dll
	new file:   update/PyQt5/qt-plugins/mediaservice/qtmedia_audioengine.dll
	new file:   update/PyQt5/qt-plugins/mediaservice/wmfengine.dll
	new file:   update/PyQt5/qt-plugins/platforms/qminimal.dll
	new file:   update/PyQt5/qt-plugins/platforms/qoffscreen.dll
	new file:   update/PyQt5/qt-plugins/platforms/qwebgl.dll
	new file:   update/PyQt5/qt-plugins/platforms/qwindows.dll
	new file:   update/PyQt5/qt-plugins/platformthemes/qxdgdesktopportal.dll
	new file:   update/PyQt5/qt-plugins/printsupport/windowsprintersupport.dll
	new file:   update/PyQt5/qt-plugins/styles/qwindowsvistastyle.dll
	new file:   update/PyQt5/sip.pyd
	new file:   update/_bz2.pyd
	new file:   update/_ctypes.pyd
	new file:   update/_decimal.pyd
	new file:   update/_elementtree.pyd
	new file:   update/_hashlib.pyd
	new file:   update/_hashlib.zip
	new file:   update/_lzma.pyd
	new file:   update/_socket.pyd
	new file:   update/libcrypto-1_1-x64.dll
	new file:   update/libeay32.dll
	new file:   update/libffi-7.dll
	new file:   update/msvcp140.dll
	new file:   update/msvcp140_1.dll
	new file:   update/psutil/_psutil_windows.pyd
	new file:   update/pyexpat.pyd
	new file:   update/python3.dll
	new file:   update/python39.dll
	new file:   update/qt5core.dll
	new file:   update/qt5dbus.dll
	new file:   update/qt5gui.dll
	new file:   update/qt5multimedia.dll
	new file:   update/qt5network.dll
	new file:   update/qt5printsupport.dll
	new file:   update/qt5qml.dll
	new file:   update/qt5qmlmodels.dll
	new file:   update/qt5quick.dll
	new file:   update/qt5svg.dll
	new file:   update/qt5websockets.dll
	new file:   update/qt5widgets.dll
	new file:   update/select.pyd
	new file:   update/ssleay32.dll
	new file:   update/unicodedata.pyd
	new file:   update/update.exe
	new file:   update/vcruntime140.dll
	new file:   update/vcruntime140_1.dll
2026-04-19 15:39:46 +08:00
铭坤
3ffbb2b004 modified: ali_oss.py
modified:   config.py
	modified:   main.py
	new file:   winsrc.bat
2026-04-19 15:38:09 +08:00
super
34ed15a9bd Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-19 15:06:13 +08:00
super
4b6295dd44 改造后台权限和APP权限 2026-04-19 15:06:09 +08:00
supernijia
01116d5607 删除文件 desktop 2026-04-18 16:23:13 +00:00
铭坤
07904a29aa modified: amazon/match_action.py
modified:   amazon/price_match.py
	modified:   config.py
2026-04-19 00:18:28 +08:00
铭坤
8d683a791d Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-19 00:18:02 +08:00
super
ef0e0df0ac 完成匹配、跟价、权限部分 2026-04-17 12:52:16 +08:00
铭坤
ac07416352 modified: amazon/__pycache__/approve.cpython-39.pyc
modified:   amazon/__pycache__/del_brand.cpython-39.pyc
	modified:   amazon/__pycache__/match_action.cpython-39.pyc
	modified:   amazon/approve.py
	modified:   amazon/del_brand.py
	modified:   amazon/match_action.py
	new file:   amazon/price_match.py
	modified:   main.py
2026-04-17 10:04:24 +08:00
super
025ca6d4fd 后台管理跳过asin、后台相关数据做数据隔离 2026-04-14 10:04:50 +08:00
951a353881 提交任务更新 2026-04-13 20:07:00 +08:00
super
eb04caccf1 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-13 13:02:45 +08:00
super
cd84def61d 提交工作更新 2026-04-13 13:02:41 +08:00
铭坤
d8098b0378 modified: app/amazon/__pycache__/del_brand.cpython-39.pyc
modified:   app/amazon/__pycache__/match_action.cpython-39.pyc
	modified:   app/amazon/del_brand.py
	modified:   app/amazon/match_action.py
	modified:   app/assets/delete-brand.js
	modified:   app/new_web_source/delete-brand.html
	new file:   "app/web_source/brand - \345\211\257\346\234\254.html"
2026-04-12 20:52:26 +08:00
super
62d30ec190 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-12 17:15:54 +08:00
super
d0d3c6ee67 完成菜单权限添加和软件菜单改造 2026-04-12 17:15:15 +08:00
铭坤
21b6b5b270 modified: app/amazon/__pycache__/approve.cpython-39.pyc
modified:   app/amazon/__pycache__/del_brand.cpython-39.pyc
	modified:   app/amazon/__pycache__/main.cpython-39.pyc
	modified:   app/amazon/__pycache__/match_action.cpython-39.pyc
	modified:   app/amazon/__pycache__/tool.cpython-39.pyc
	modified:   app/amazon/approve.py
	modified:   app/amazon/del_brand.py
	modified:   app/amazon/main.py
	modified:   app/amazon/match_action.py
	modified:   app/amazon/tool.py
	modified:   app/assets/delete-brand.js
	modified:   app/new_web_source/delete-brand.html
2026-04-11 23:00:55 +08:00
super
365050b890 提交bug修复 后台权限管理 2026-04-11 22:25:14 +08:00
super
73392a9b83 修复BUG,完善匹配店铺 2026-04-11 00:57:34 +08:00
super
c47c03fde4 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-10 19:35:08 +08:00
super
86fd9475e4 匹配回收,bug修复 2026-04-10 19:33:58 +08:00
铭坤
2d0d1d3461 modified: app/amazon/__pycache__/approve.cpython-39.pyc
modified:   app/amazon/__pycache__/del_brand.cpython-39.pyc
	modified:   app/amazon/__pycache__/main.cpython-39.pyc
	new file:   app/amazon/__pycache__/match_action.cpython-39.pyc
	new file:   app/amazon/__pycache__/tool.cpython-39.pyc
	new file:   "app/amazon/approve - \345\211\257\346\234\254.py"
	modified:   app/amazon/approve.py
	modified:   app/amazon/del_brand.py
	modified:   app/amazon/main.py
	new file:   app/amazon/match_action.py
	new file:   app/amazon/tool.py
	modified:   app/main.py
	modified:   app/web_source/brand.html
2026-04-10 16:51:52 +08:00
super
ccaec4a984 完善后台管理店铺搜索 2026-04-08 21:08:52 +08:00
super
169ba7edb6 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-08 16:14:30 +08:00
super
0327d1cc51 修复接口内容缺失 2026-04-08 16:14:26 +08:00
铭坤
e02ddab599 modified: amazon/__pycache__/approve.cpython-39.pyc
modified:   amazon/__pycache__/del_brand.cpython-39.pyc
	modified:   amazon/__pycache__/main.cpython-39.pyc
	modified:   amazon/approve.py
	new file:   amazon/backend_api.py
	modified:   amazon/del_brand.py
	modified:   amazon/main.py
	modified:   blueprints/main.py
	modified:   main.py
2026-04-08 16:03:48 +08:00
super
336338cff5 完善后台接口字段 2026-04-07 17:57:03 +08:00
super
acff31e652 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-07 14:50:09 +08:00
super
ad304a6880 完成店铺增删改查 2026-04-07 14:50:06 +08:00
铭坤
cb34788e53 modified: "web_source/brand-\346\227\247.html"
modified:   web_source/brand.html
	modified:   web_source/templates_backup/brand.html
2026-04-07 14:10:33 +08:00
铭坤
13b0ffb5d8 new file: amazon/__pycache__/approve.cpython-39.pyc
modified:   amazon/__pycache__/del_brand.cpython-39.pyc
	modified:   amazon/__pycache__/main.cpython-39.pyc
	new file:   amazon/approve.py
	modified:   amazon/del_brand.py
	modified:   amazon/main.py
	modified:   main.py
2026-04-07 14:09:42 +08:00
super
c9947c4ac8 更改下载方法名 2026-04-06 22:26:44 +08:00
super
afffb7aad2 更改下载类型处理 2026-04-06 16:58:58 +08:00
super
9720453ac4 更新商品风险处理 2026-04-05 22:04:21 +08:00
super
1ff6d5220e 新增删除增加公司名 2026-04-04 00:10:08 +08:00
super
46d91fd8ca 提交接口进度完善 2026-04-03 00:53:52 +08:00
super
a24db2f73c Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-02 23:29:19 +08:00
super
ca9537f862 测试多任务暂存 2026-04-02 23:29:15 +08:00
铭坤
94ae7e68ff modified: app/amazon/__pycache__/del_brand.cpython-39.pyc
modified:   app/amazon/__pycache__/main.cpython-39.pyc
	modified:   app/amazon/main.py
	modified:   app/main.py
2026-04-02 23:03:13 +08:00
super
94e1930538 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-02 20:37:38 +08:00
super
5bbe5a3077 修改多文件 2026-04-02 20:37:35 +08:00
铭坤
8cead01270 modified: app/amazon/del_brand.py
backend/blueprints/get_resource.py
2026-04-02 20:35:50 +08:00
铭坤
99fc14b9f9 modified: backend/app.py
backend/blueprints/get_resource.py
2026-04-02 20:35:12 +08:00
super
d3671206e1 完善后端轮询记忆紫鸟店铺逻辑 2026-04-02 12:26:48 +08:00
super
0a8202788b Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-02 01:15:54 +08:00
super
855f1affd9 紫鸟查询店铺更新 2026-04-02 01:15:50 +08:00
铭坤
5a294e5c85 new file: app/amazon/__pycache__/del_brand.cpython-39.pyc
new file:   app/amazon/__pycache__/main.cpython-39.pyc
	modified:   app/amazon/del_brand.py
	modified:   app/amazon/main.py
	modified:   app/blueprints/__pycache__/main.cpython-39.pyc
	modified:   app/blueprints/main.py
	modified:   app/config.py
	modified:   app/main.py
	new file:   "app/web_source/brand-\346\227\247.html"
	modified:   app/web_source/brand.html
2026-04-02 00:03:16 +08:00
super
df606a1087 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-01 21:20:04 +08:00
super
ece3552b89 更新部分内容 2026-04-01 21:20:01 +08:00
铭坤
4df3945131 modified: app/ali_oss.py
new file:   app/amazon/del_brand.py
	new file:   app/amazon/main.py
	modified:   app/blueprints/__pycache__/brand.cpython-39.pyc
	modified:   app/blueprints/__pycache__/communication.cpython-39.pyc
	modified:   app/blueprints/__pycache__/main.cpython-39.pyc
	modified:   app/blueprints/brand.py
	new file:   "app/blueprints/brand_\345\244\207\344\273\275.py"
	modified:   app/blueprints/communication.py
	modified:   app/blueprints/main.py
	modified:   app/config.py
	modified:   app/main.py
2026-04-01 11:53:12 +08:00
super
6eb0f33421 删除模块 取消打开紫鸟 2026-03-31 23:12:46 +08:00
super
25ce9f74b8 处理这个python打开紫鸟 2026-03-30 23:15:08 +08:00
super
0f1f471590 处理掉一些BUG 2026-03-30 22:47:19 +08:00
super
d9487b6885 品牌多文件chunk修复 2026-03-30 21:06:10 +08:00
super
9845ab8f2a 暂存 2026-03-30 20:54:05 +08:00
super
6fa130386a Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-03-30 20:48:02 +08:00
super
53df8b9971 暂存 2026-03-30 20:39:13 +08:00
铭坤
d6625762cf modified: app/.env
modified:   app/ali_oss.py
	modified:   app/app.py
	new file:   app/blueprints/__pycache__/communication.cpython-39.pyc
	modified:   app/blueprints/communication.py
	modified:   app/brand_spider/__pycache__/main.cpython-39.pyc
	modified:   app/config.py
	modified:   app/main.py
	modified:   app/static/bg.jpg
	modified:   app/tool/__pycache__/devices.cpython-39.pyc
	modified:   app/web_source/brand.html
	modified:   app/web_source/templates_backup/brand.html
2026-03-30 19:44:04 +08:00
super
cb3b9e9cfe 完善删除品牌asin 2026-03-29 19:03:24 +08:00
super
3d39236b13 完成删除品牌 2026-03-29 13:41:03 +08:00
super
9e57fa5762 完成品牌删除模块 2026-03-29 00:22:45 +08:00
super
9c6232bc85 完成前后端紫鸟部分开发 2026-03-28 15:42:18 +08:00
super
d3c7938627 完成前后端删除、紫鸟部分开发 2026-03-28 15:42:02 +08:00
super
06256946fa 提交暂存 2026-03-27 22:00:01 +08:00
super
2042d386e2 提交过滤 2026-03-27 21:58:43 +08:00
铭坤
7176754564 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-03-27 21:56:53 +08:00
铭坤
a20222192e modified: assets/convert.js
modified:   assets/dedupe.js
	modified:   assets/split.js
	modified:   blueprints/__pycache__/brand.cpython-311.pyc
	modified:   blueprints/__pycache__/brand.cpython-39.pyc
	new file:   blueprints/communication.py
	modified:   config.py
	modified:   main.py
	modified:   new_web_source/convert.html
	modified:   new_web_source/dedupe.html
	modified:   new_web_source/split.html
2026-03-27 21:56:19 +08:00
super
a5c2681e88 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-03-27 21:51:34 +08:00
super
5e2f750359 更新运行日志
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-27 21:51:23 +08:00
super
27602890ab 品牌 2026-03-27 21:48:13 +08:00
铭坤
dfe0d652ce new file: assets/_plugin-vue_export-helper-Dh4yMZWw.js
new file:   assets/_plugin-vue_export-helper-xvHHTGU_.css
	new file:   assets/convert-1dkFPbKa.css
	new file:   assets/convert-6b0yq-M0.css
	new file:   assets/convert-BxkpMHO9.css
	new file:   assets/convert-D4Yv2oRy.css
	new file:   assets/convert-DwbQug2y.css
	modified:   assets/convert.js
	new file:   assets/dedupe-B0eqpVpj.css
	new file:   assets/dedupe-BHE_BCAJ.css
	new file:   assets/dedupe-Day_nGxq.css
	new file:   assets/dedupe-DzFvTfoj.css
	new file:   assets/dedupe-rhGu1E8E.css
	modified:   assets/dedupe.js
	new file:   assets/el-alert-Ct0RpqUD.css
	new file:   assets/pywebview-B6ZNXusn.css
	new file:   assets/pywebview-BRrok0pS.js
	new file:   assets/pywebview-BnpJyyGZ.js
	new file:   assets/pywebview-CGwF8TuM.js
	new file:   assets/pywebview-Chz98E9c.css
	new file:   assets/pywebview-D-PKWjUg.js
	new file:   assets/pywebview-DVgP2-kW.js
	new file:   assets/pywebview-IqgMfeBe.css
	new file:   assets/pywebview-Rm5Hhqpf.js
	new file:   assets/pywebview-leZCedEt.js
	new file:   assets/split-BsJfbBqn.css
	new file:   assets/split-DAa_Usoh.css
	new file:   assets/split-DidKJ84l.css
	new file:   assets/split-R_8V5xgh.css
	new file:   assets/split-_O6NHqDj.css
	modified:   assets/split.js
	modified:   blueprints/__pycache__/brand.cpython-311.pyc
	modified:   blueprints/__pycache__/brand.cpython-39.pyc
	modified:   blueprints/brand.py
	modified:   config.py
	modified:   main.py
	modified:   new_web_source/convert.html
	modified:   new_web_source/dedupe.html
	modified:   new_web_source/split.html
2026-03-27 21:35:28 +08:00
super
24076adf98 品牌 2026-03-27 21:33:07 +08:00
super
15a78abc1b 删除品牌 2026-03-27 21:32:00 +08:00
super
7fcdf1a761 提交品牌删除更新 2026-03-26 17:39:20 +08:00
super
d0ca28ae2d 提交品牌删除更新 2026-03-26 17:39:01 +08:00
super
32a9494dd6 更新后端新增内容 2026-03-26 16:42:25 +08:00
铭坤
a5da06537e modified: blueprints/__pycache__/__init__.cpython-39.pyc
modified:   blueprints/__pycache__/admin.cpython-39.pyc
	modified:   blueprints/__pycache__/auth.cpython-39.pyc
	modified:   blueprints/__pycache__/brand.cpython-311.pyc
	modified:   blueprints/__pycache__/brand.cpython-39.pyc
	modified:   blueprints/brand.py
	new file:   brand_spider/__pycache__/main.cpython-311.pyc
	modified:   brand_spider/__pycache__/main.cpython-39.pyc
	new file:   brand_spider/__pycache__/web_dec.cpython-311.pyc
	modified:   brand_spider/main.py
	modified:   config.py
	modified:   web_source/admin.html
	modified:   web_source/brand.html
	modified:   web_source/home.html
	modified:   web_source/index.html
	modified:   web_source/login.html
	new file:   web_source/templates_backup/admin.html
	new file:   web_source/templates_backup/brand.html
	new file:   web_source/templates_backup/home.html
	new file:   web_source/templates_backup/index.html
	new file:   web_source/templates_backup/login.html
2026-03-26 16:39:39 +08:00
super
21c6f41c69 增加紫鸟相关接口 2026-03-25 21:33:49 +08:00
super
506eb0faef 完善打包部分的压缩包 2026-03-25 09:52:04 +08:00
super
485b4dd485 完成品牌部分改造 2026-03-24 22:22:37 +08:00
super
4f728107d1 增加用户访问数据总表 2026-03-22 21:27:41 +08:00
super
01e491e332 数据去重完善 2026-03-22 20:58:36 +08:00
super
ed91c2ae7d 总表更新 2026-03-22 17:51:34 +08:00
super
55ac81f0cc 修复tab切换问题 2026-03-22 13:39:32 +08:00
super
64e32523cd 完成后台管理开发 2026-03-22 13:33:37 +08:00
super
5b636c3dfd 提交所有内容 2026-03-22 12:45:58 +08:00
super
c5d62c71a3 暂替 2026-03-22 11:37:23 +08:00
super
fa384a6ec8 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-03-22 11:37:03 +08:00
super
24cf47e1e3 暂提更新 2026-03-22 11:35:45 +08:00
super
8e66926b9e 更新新增三个模块 2026-03-22 11:20:09 +08:00
铭坤
574b372f42 new file: app/assets/_plugin-vue_export-helper-Dh4yMZWw.js
new file:   app/assets/_plugin-vue_export-helper-xvHHTGU_.css
	new file:   app/assets/convert-1dkFPbKa.css
	new file:   app/assets/convert-BxkpMHO9.css
	new file:   app/assets/convert-DwbQug2y.css
	modified:   app/assets/convert.js
	new file:   app/assets/dedupe-B0eqpVpj.css
	new file:   app/assets/dedupe-DzFvTfoj.css
	new file:   app/assets/dedupe-rhGu1E8E.css
	modified:   app/assets/dedupe.js
	new file:   app/assets/el-alert-Ct0RpqUD.css
	new file:   app/assets/pywebview-B6ZNXusn.css
	new file:   app/assets/pywebview-BnpJyyGZ.js
	new file:   app/assets/pywebview-CGwF8TuM.js
	new file:   app/assets/pywebview-D-PKWjUg.js
	new file:   app/assets/pywebview-IqgMfeBe.css
	new file:   app/assets/split-DAa_Usoh.css
	new file:   app/assets/split-DidKJ84l.css
	new file:   app/assets/split-R_8V5xgh.css
	modified:   app/assets/split.js
	modified:   app/blueprints/__pycache__/main.cpython-39.pyc
	modified:   app/blueprints/main.py
	modified:   app/main.py
	modified:   app/new_web_source/convert.html
	modified:   app/new_web_source/dedupe.html
	modified:   app/new_web_source/split.html
2026-03-22 10:29:12 +08:00
super
74a5fb9ce1 增加log输出、跳转tab顺序 2026-03-21 20:34:49 +08:00
super
4b83bf76b4 环境搭建配置 2026-03-21 10:00:16 +08:00
铭坤
82ccd9a8ea Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-03-20 23:16:13 +08:00
铭坤
2dbeec7d6d new file: app/blueprints/__init__.py
new file:   app/blueprints/__pycache__/__init__.cpython-311.pyc
	new file:   app/blueprints/__pycache__/__init__.cpython-39.pyc
	new file:   app/blueprints/__pycache__/admin.cpython-311.pyc
	new file:   app/blueprints/__pycache__/admin.cpython-39.pyc
	new file:   app/blueprints/__pycache__/auth.cpython-311.pyc
	new file:   app/blueprints/__pycache__/auth.cpython-39.pyc
	new file:   app/blueprints/__pycache__/brand.cpython-311.pyc
	new file:   app/blueprints/__pycache__/brand.cpython-39.pyc
	new file:   app/blueprints/__pycache__/image.cpython-311.pyc
	new file:   app/blueprints/__pycache__/image.cpython-39.pyc
	new file:   app/blueprints/__pycache__/main.cpython-311.pyc
	new file:   app/blueprints/__pycache__/main.cpython-39.pyc
	new file:   app/blueprints/admin.py
	new file:   app/blueprints/auth.py
	new file:   app/blueprints/brand.py
	new file:   app/blueprints/image.py
	new file:   app/blueprints/main.py
	new file:   app/brand_spider/__pycache__/main.cpython-39.pyc
	new file:   app/brand_spider/__pycache__/web_dec.cpython-39.pyc
	new file:   app/brand_spider/main.py
	new file:   app/brand_spider/web_dec.py
	new file:   app/static/bg.jpg
	new file:   "app/static/\345\223\201\347\211\214\346\226\207\346\241\243\346\240\274\345\274\217_\346\250\241\346\235\277.xlsx"
	new file:   "app/static/\346\250\241\346\235\2772-\344\273\245\346\226\207\344\273\266\345\244\271\346\226\271\345\274\217\344\270\212\344\274\240.zip"
	new file:   app/tool/.device_id
	new file:   app/tool/__pycache__/devices.cpython-311.pyc
	new file:   app/tool/__pycache__/devices.cpython-39.pyc
	new file:   app/tool/devices.py
2026-03-20 23:04:19 +08:00
super
fb32f7bd35 新增接口文档 2026-03-20 19:31:02 +08:00
super
63d3e4047c 打包项目 2026-03-20 18:13:47 +08:00
super
7c6d9dd989 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-03-20 14:26:41 +08:00
铭坤
ce121cf0d0 new file: ali_oss.py
new file:   app.py
	new file:   app/ali_oss.py
	new file:   app/app.py
	new file:   app/app_common.py
	new file:   app/config.py
	new file:   app/coze.py
	new file:   app/generate_api.py
	new file:   app/html_crypto.py
	new file:   app/main.py
	new file:   app/web_source/admin.html
	new file:   app/web_source/brand.html
	new file:   app/web_source/home.html
	new file:   app/web_source/index.html
	new file:   app/web_source/login.html
	new file:   app_common.py
	new file:   blueprints/__init__.py
	new file:   blueprints/__pycache__/__init__.cpython-311.pyc
	new file:   blueprints/__pycache__/__init__.cpython-39.pyc
	new file:   blueprints/__pycache__/admin.cpython-311.pyc
	new file:   blueprints/__pycache__/admin.cpython-39.pyc
	new file:   blueprints/__pycache__/auth.cpython-311.pyc
	new file:   blueprints/__pycache__/auth.cpython-39.pyc
	new file:   blueprints/__pycache__/brand.cpython-311.pyc
	new file:   blueprints/__pycache__/brand.cpython-39.pyc
	new file:   blueprints/__pycache__/image.cpython-311.pyc
	new file:   blueprints/__pycache__/image.cpython-39.pyc
	new file:   blueprints/__pycache__/main.cpython-311.pyc
	new file:   blueprints/__pycache__/main.cpython-39.pyc
	new file:   blueprints/admin.py
	new file:   blueprints/auth.py
	new file:   blueprints/brand.py
	new file:   blueprints/image.py
	new file:   blueprints/main.py
	new file:   brand_spider/__pycache__/main.cpython-39.pyc
	new file:   brand_spider/__pycache__/web_dec.cpython-39.pyc
	new file:   brand_spider/main.py
	new file:   brand_spider/web_dec.py
	new file:   config.py
	new file:   coze.py
	new file:   generate_api.py
	new file:   html_crypto.py
	new file:   main.py
	new file:   static/bg.jpg
	new file:   "static/\345\223\201\347\211\214\346\226\207\346\241\243\346\240\274\345\274\217_\346\250\241\346\235\277.xlsx"
	new file:   "static/\346\250\241\346\235\2772-\344\273\245\346\226\207\344\273\266\345\244\271\346\226\271\345\274\217\344\270\212\344\274\240.zip"
	new file:   tool/.device_id
	new file:   tool/__pycache__/devices.cpython-311.pyc
	new file:   tool/__pycache__/devices.cpython-39.pyc
	new file:   tool/devices.py
2026-03-20 11:23:57 +08:00
514 changed files with 45684 additions and 57263 deletions

2
.gitignore vendored
View File

@@ -43,6 +43,7 @@ MANIFEST
.installed.cfg
# Build / packaging
app.zip
build/
dist/
develop-eggs/
@@ -108,3 +109,4 @@ xlsx/
OPS_REDIS_MYSQL_OPTIMIZATION_NOTES.md
架构.md
*ts.%
.omc

View File

@@ -1,20 +0,0 @@
base_url=http://8.136.19.173:15124
workflow_id=7608812635877900322
mysql_host=8.136.19.173
mysql_user=aiimage
proxy_url=https://api.jikip.com/ip-get?num=1&minute=1&format=json&area=all&protocol=1&mode=2&key=t24g6gi44ubufd8
proxy_mode=2
zn_company=rongchuang123
zn_username=%E8%87%AA%E5%8A%A8%E5%8C%96_Robot
client_name=ShuFuAI
# java_api_base=http://47.111.163.154:18080
# java_api_base=http://127.0.0.1:18080
# java_api_base=http://8.136.19.173:18080
java_api_base=http://121.196.149.225:18080

View File

@@ -1,15 +0,0 @@
base_url=http://8.136.19.173:15124
workflow_id=7608812635877900322
mysql_host=8.136.19.173
mysql_user=aiimage
proxy_url=https://api.jikip.com/ip-get?num=1&minute=1&format=json&area=all&protocol=1&mode=2&key=t24g6gi44ubufd8
proxy_mode=2
client_name=ShuFuAI
java_api_base=http://127.0.0.1:18080
# java_api_base=http://8.136.19.173:18080

View File

@@ -1,18 +0,0 @@
workflow_id=7608812635877900322
mysql_host=8.136.19.173
mysql_user=aiimage
proxy_url=https://api.jikip.com/ip-get?num=1&minute=1&format=json&area=all&protocol=1&mode=2&key=t24g6gi44ubufd8
proxy_mode=2
zn_company=rongchuang123
zn_username=%E8%87%AA%E5%8A%A8%E5%8C%96_Robot
client_name=ShuFuAI
java_api_base=http://47.111.163.154:18080
# java_api_base=http://127.0.0.1:18080
# java_api_base=http://8.136.19.173:18080

View File

@@ -1,72 +0,0 @@
import argparse
import base64
import re
import time
import alibabacloud_oss_v2 as oss
import requests
from config import region, endpoint, bucket, file_url_pre, bucket_path
def upload_file(file_content: bytes, key: str):
credentials_provider = oss.credentials.EnvironmentVariableCredentialsProvider()
cfg = oss.config.load_default()
cfg.credentials_provider = credentials_provider
cfg.region = region
cfg.endpoint = endpoint
client = oss.Client(cfg)
result = client.put_object(
oss.PutObjectRequest(
bucket=bucket, # 存储空间名称
key=key, # 对象名称
body=file_content # 读取文件内容
)
)
# print(result)
return file_url_pre + key
def upload_data_url(data_url: str, prefix: str = "history", key_hint: str = "") -> str:
"""
将 base64 data URL 上传到 OSS返回图片链接
data_url: data:image/png;base64,xxxx 或 data:image/jpeg;base64,xxxx
prefix: OSS key 前缀
key_hint: 可选后缀避免重名,如 "_0", "_1"
"""
match = re.match(r'data:image/(\w+);base64,(.+)', data_url)
if not match:
raise ValueError('无效的 data URL 格式')
ext = 'png' if match.group(1).lower() in ('png', 'webp') else 'jpg'
file_content = base64.b64decode(match.group(2))
ts = int(time.time() * 1000)
key = f"{bucket_path}{prefix}/{ts}{key_hint}.{ext}"
return upload_file(file_content, key)
def upload_data_urls(data_urls: list, prefix: str = "history") -> list:
"""批量上传 base64 图片到 OSS返回图片链接列表"""
urls = []
ts = int(time.time() * 1000)
for i, data_url in enumerate(data_urls or []):
if not data_url or not isinstance(data_url, str):
continue
if not data_url.startswith("http"):
match = re.match(r'data:image/(\w+);base64,(.+)', data_url)
if not match:
continue
ext = 'png' if match.group(1).lower() in ('png', 'webp') else 'jpg'
file_content = base64.b64decode(match.group(2))
else:
file_content = requests.get(data_url).content
ext = "png"
key = f"{bucket_path}{prefix}/{ts}_{i}.{ext}"
urls.append(upload_file(file_content, key))
return urls
# 脚本入口当文件被直接运行时调用main函数
if __name__ == "__main__":
with open("测试图片数据/IMG_2685.JPG", "rb") as f:
file_content = f.read()
upload_file(file_content,key=bucket_path+"test.png")

View File

@@ -1,847 +0,0 @@
import traceback
import winreg
import subprocess
import time
import uuid
import requests
import json
import os
import re
from datetime import datetime
from DrissionPage import Chromium,ChromiumOptions
from DrissionPage.common import By
from DrissionPage._pages.chromium_tab import ChromiumTab
from typing import Literal,Dict,Any,TypeVar,Type
from amazon.tool import get_shop_info,show_notification
from config import runing_shop
# 定义类型变量,用于泛型类型注解
T = TypeVar('T', bound='AmamzonBase')
def kill_process(version: Literal["v5", "v6"]):
"""杀紫鸟客户端进程(独立函数版本)"""
driver = ZiniaoDriver({})
driver.kill_process(version)
class ZiniaoDriver:
"""紫鸟浏览器自动化驱动类"""
def __init__(self, user_info: dict, socket_port: int = 19890):
"""
初始化紫鸟浏览器驱动
Args:
user_info: 用户信息字典,包含 company, username, password
socket_port: 客户端通信端口,默认 19890
"""
self.user_info = user_info
self.socket_port = socket_port
self.client_path = None
self.browser = None
self.tab: ChromiumTab = None
self.store_id = None
self.url = None #用于记录当前链接,实现断点续传
def get_zinaio_exe(self, protocol_name: str = "superbrowser"):
"""
获取紫鸟安装目录
Args:
protocol_name: 协议名称,默认 "superbrowser"
Returns:
exe_path: 紫鸟浏览器可执行文件路径
"""
try:
key_path = rf"SOFTWARE\Classes\{protocol_name}\shell\open\command"
key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, key_path)
command, _ = winreg.QueryValueEx(key, "")
winreg.CloseKey(key)
if isinstance(command, str):
sub = "ziniao.exe"
exe_path = command[0: command.find(sub) + len(sub) + 1]
else:
exe_path = command[0]
return exe_path
except FileNotFoundError:
try:
key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, key_path)
command, _ = winreg.QueryValueEx(key, "")
winreg.CloseKey(key)
if isinstance(command, str):
sub = "ziniao.exe"
exe_path = command[0: command.find(sub) + len(sub) + 1]
else:
exe_path = command[0]
return exe_path
except FileNotFoundError:
return None
def update_core(self):
"""
下载所有内核打开店铺前调用需客户端版本5.285.7以上
因为http有超时时间所以这个action适合循环调用直到返回成功
"""
data = {
"action": "updateCore",
"requestId": str(uuid.uuid4()),
}
data.update(self.user_info)
while True:
url = f'http://127.0.0.1:{self.socket_port}'
response = requests.post(url, json.dumps(data).encode('utf-8'), timeout=120)
result = response.json()
print(result)
if result is None:
print("等待客户端启动...")
time.sleep(2)
continue
if result.get("statusCode") is None or result.get("statusCode") == -10003:
print("当前版本不支持此接口,请升级客户端")
return
elif result.get("statusCode") == 0:
print("更新内核完成")
return
else:
print(f"等待更新内核: {json.dumps(result)}")
time.sleep(2)
def kill_process(self, version: Literal["v5", "v6"]):
"""
杀紫鸟客户端进程
Args:
version: 客户端版本
"""
if version == "v5":
process_name = 'SuperBrowser.exe'
os.system('taskkill /f /t /im ' + "starter.exe")
else:
process_name = 'ziniao.exe'
os.system('taskkill /f /t /im ' + process_name)
time.sleep(3)
def get_browser_list(self) -> list:
"""
获取浏览器列表
Returns:
list: 浏览器列表
"""
request_id = str(uuid.uuid4())
data = {
"action": "getBrowserList",
"requestId": request_id
}
data.update(self.user_info)
url = f'http://127.0.0.1:{self.socket_port}'
response = requests.post(url, json.dumps(data).encode('utf-8'), timeout=120)
r = response.json()
if str(r.get("statusCode")) == "0":
print(r)
return r.get("browserList")
elif str(r.get("statusCode")) == "-10003":
print(f"【get_browser_list】登录失败 {json.dumps(r, ensure_ascii=False)}")
# exit()
else:
print(f"【get_browser_list】失败 {json.dumps(r, ensure_ascii=False)} ")
# exit()
def open_store(self, store_info, isWebDriverReadOnlyMode=0, isprivacy=0,
isHeadless=0, cookieTypeSave=0, jsInfo=""):
"""
打开店铺
Args:
store_info: 店铺信息browserId 或 browserOauth
isWebDriverReadOnlyMode: 是否只读模式,默认 0
isprivacy: 隐私模式,默认 0
isHeadless: 无头模式,默认 0
cookieTypeSave: cookie保存类型默认 0
jsInfo: 注入的JS信息默认 ""
Returns:
dict: 返回结果
"""
request_id = str(uuid.uuid4())
data = {
"action": "startBrowser",
"isWaitPluginUpdate": 0,
"isHeadless": isHeadless,
"requestId": request_id,
"isWebDriverReadOnlyMode": isWebDriverReadOnlyMode,
"cookieTypeLoad": 0,
"cookieTypeSave": cookieTypeSave,
"runMode": "1",
"isLoadUserPlugin": True,
"pluginIdType": 1,
"privacyMode": isprivacy
}
data.update(self.user_info)
if store_info.isdigit():
data["browserId"] = store_info
else:
data["browserOauth"] = store_info
if len(str(jsInfo)) > 2:
data["injectJsInfo"] = json.dumps(jsInfo)
url = f'http://127.0.0.1:{self.socket_port}'
response = requests.post(url, json.dumps(data).encode('utf-8'), timeout=120)
r = response.json()
if str(r.get("statusCode")) == "0":
return r
elif str(r.get("statusCode")) == "-10003":
raise RuntimeError(f"【open_store】登录失败 {json.dumps(r, ensure_ascii=False)}")
# exit()
else:
raise RuntimeError(f"【open_store】失败 {json.dumps(r, ensure_ascii=False)} ")
# exit()
def get_browser(self, port) -> Chromium:
"""
获取浏览器实例
Args:
port: 调试端口
Returns:
Chromium: DrissionPage浏览器实例
"""
co = ChromiumOptions()
# 设置不加载图片、静音
co.set_local_port(port)
co.no_imgs(True).mute(True)
browser = Chromium(co)
return browser
def start_client(self):
"""启动紫鸟客户端"""
# 检查端口是否已经启动
try:
url = f'http://127.0.0.1:{self.socket_port}'
response = requests.get(url, timeout=2)
print(f"端口 {self.socket_port} 已经启动,跳过启动客户端操作")
return
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
# 端口未启动,继续执行启动操作
print(f"端口 {self.socket_port} 未启动,开始启动客户端")
self.kill_process('v6')
time.sleep(5)
self.client_path = self.get_zinaio_exe("superbrowserv6").strip('"')
print(self.client_path)
cmd = [self.client_path, '--run_type=web_driver', '--ipc_type=http',
'--port=' + str(self.socket_port)]
print(" ".join(cmd))
# 最大重试次数
max_retries = 3
for retry_count in range(max_retries):
print(f"{retry_count + 1} 次尝试启动客户端...")
# 启动进程
subprocess.Popen(cmd)
# 循环检测10秒每0.5秒检测一次
start_check_time = time.time()
client_started = False
while time.time() - start_check_time < 10:
try:
response = requests.get(url, timeout=2)
print(f"客户端启动成功!(第 {retry_count + 1} 次尝试)")
client_started = True
break
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
# 端口还未启动,继续等待
time.sleep(0.5)
if client_started:
time.sleep(5) # 等待客户端完全启动
# 更新内核
self.update_core()
return
else:
print(f"{retry_count + 1} 次尝试启动失败10秒内未检测到客户端启动")
# 超过最大重试次数,抛出异常
raise RuntimeError(f"客户端启动失败:重试 {max_retries} 次后仍未成功启动")
def open_shop(self, shop_name: str):
"""
打开指定店铺
Args:
shop_name: 店铺名称
Returns:
Chromium or str: 成功返回浏览器实例,失败返回错误信息
"""
print("=============打开指定店铺================")
# 启动客户端
self.start_client()
# 获取店铺列表
shop_ls = self.get_browser_list()
self.store_id = None
print(shop_ls)
for shop in shop_ls:
if shop.get("browserName") == shop_name:
self.store_id = shop.get('browserOauth')
break
if not self.store_id:
print("店铺不存在")
return "店铺不存在"
# 打开店铺
ret_json = self.open_store(self.store_id)
print(ret_json)
self.store_id = ret_json.get("browserOauth")
if self.store_id is None:
self.store_id = ret_json.get("browserId")
# 获取drissionpage浏览器会话
self.browser = self.get_browser(ret_json.get('debuggingPort'))
ip_check_url = ret_json.get("ipDetectionPage")
if not ip_check_url:
print("ip检测页地址为空请升级紫鸟浏览器到最新版")
print(f"=====关闭店铺:{shop_name}=====")
self.close_store(self.store_id)
# exit()
raise RuntimeError("没有IP检测地址为了店铺安全不打开店铺")
ip_usable = self.open_ip_check(self.browser, ip_check_url)
if ip_usable:
print("ip检测通过打开店铺平台主页")
self.open_launcher_page(ret_json.get("launcherPage"), self.browser)
else:
print("IP检测不通过")
raise RuntimeError("IP检测不通过可能是因为网络环境变化导致的为了店铺安全不打开店铺")
return self.browser
def close_store(self, browser_oauth=None):
"""
关闭店铺
Args:
browser_oauth: 店铺OAuth标识如果不提供则使用当前打开的店铺
Returns:
dict: 返回结果
"""
if browser_oauth is None:
browser_oauth = self.store_id
request_id = str(uuid.uuid4())
data = {
"action": "stopBrowser",
"requestId": request_id,
"duplicate": 0,
"browserOauth": browser_oauth
}
data.update(self.user_info)
url = f'http://127.0.0.1:{self.socket_port}'
response = requests.post(url, json.dumps(data).encode('utf-8'), timeout=120)
r = response.json()
if str(r.get("statusCode")) == "0":
return r
elif str(r.get("statusCode")) == "-10003":
raise RuntimeError(f"【close_store】登录失败 {json.dumps(r, ensure_ascii=False)}")
# exit()
else:
raise RuntimeError(f"【close_store】失败 {json.dumps(r, ensure_ascii=False)} ")
# exit()
def open_launcher_page(self, launcher_page: str, browser: Chromium = None):
"""
打开启动页面
Args:
launcher_page: 要打开的页面URL
browser: 浏览器实例,如果不提供则使用当前浏览器实例
"""
if browser is None:
browser = self.browser
tab = browser.new_tab(url=launcher_page)
self.tab = tab
return tab
def open_ip_check(self, browser: Chromium, ip_check_url: str):
"""
打开ip检测页检测ip是否正常
:param browser: drissionpage浏览器会话
:param ip_check_url ip检测页地址
:return 检测结果
"""
try:
tab = browser.latest_tab
tab.get(ip_check_url)
success_button = tab.ele((By.XPATH, '//button[contains(@class, "styles_btn--success")]'),
timeout=60) # 等待查找元素60秒
if success_button:
print("ip检测成功")
return True
else:
print("ip检测超时")
return False
except Exception as e:
print("ip检测异常:" + traceback.format_exc())
return False
class AmamzonBase(ZiniaoDriver):
"""亚马逊操作基类,包含一些通用方法"""
mark_name = "操作基类"
def log(self, message: str, level: str = "INFO"):
"""日志输出
Args:
message: 日志消息
level: 日志级别
"""
try:
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if level == "ERROR":
show_notification(message, "error")
print(f"[{timestamp}] [{self.mark_name}] [{level}] {message}")
except Exception as e:
print(f"输出出错,{e}")
def SwitchingCountries(self, country_name: str):
"""
切换国家
操作:
1、//div[@class="dropdown-account-switcher-header-label"]/span[last()] 获取此元素文本,判断当前国家,如果与目标国家相同则不操作,否则执行下一步
2、点击 //div[@class="dropdown-account-switcher-header-label"] 打开下拉框
3、点击 //div[@class="dropdown-account-switcher-list-item"] 第一个展开国家列表
4、点击 //div[@class="dropdown-account-switcher-list-item dropdown-account-switcher-list-item-indented" and @title="国家名"] 切换到目标国家
5、等待页面加载完成判断国家是否切换成功成功则返回True否则返回False
Args:
country_name: 目标国家名称
Returns:
bool: 切换成功返回True失败返回False
"""
try:
print("=============切换国家================")
if self.browser is None:
print("浏览器实例不存在,请先打开店铺")
return False
# 获取当前标签页
tab = self.tab
tab.wait.doc_loaded(timeout=120, raise_err=False)
# 步骤1获取当前国家名称
self.log(f"正在检查当前国家...")
current_country_ele = tab.ele('xpath://div[@class="dropdown-account-switcher-header-label"]/span[last()]',
timeout=20)
if current_country_ele:
current_country = current_country_ele.text.strip()
self.log(f"当前国家:{current_country}")
# 判断是否与目标国家相同
if current_country == country_name:
self.log(f"当前已经是目标国家 {country_name},无需切换")
return True
else:
self.log("无法获取当前国家信息")
return False
# 步骤2点击打开下拉框
self.log(f"正在打开国家切换下拉框...")
dropdown_header = tab.ele('xpath://div[@class="dropdown-account-switcher-header-label"]', timeout=10)
if not dropdown_header:
self.log("找不到国家切换下拉框")
return False
dropdown_header.click()
time.sleep(1) # 等待下拉框展开
# 步骤3点击第一个展开国家列表
self.log(f"正在展开国家列表...")
first_item = tab.ele('xpath://div[@class="dropdown-account-switcher-list-item"]', timeout=10)
if not first_item:
self.log("找不到国家列表项")
return False
first_item.click()
time.sleep(1) # 等待国家列表展开
# 步骤4点击目标国家
self.log(f"正在切换到国家:{country_name}")
target_country_xpath = f'//div[@class="dropdown-account-switcher-list-item dropdown-account-switcher-list-item-indented" and @title="{country_name}"]'
target_country = tab.ele(f'xpath:{target_country_xpath}', timeout=10)
if not target_country:
self.log(f"找不到目标国家:{country_name}")
return False
target_country.click()
# 步骤5等待页面加载完成并验证切换结果
self.log(f"等待页面加载...")
# time.sleep(3) # 等待页面加载
self.tab.wait.doc_loaded()
# 再次检查当前国家
new_country_ele = tab.ele('xpath://div[@class="dropdown-account-switcher-header-label"]/span[last()]',
timeout=10)
if new_country_ele:
new_country = new_country_ele.text.strip()
if new_country == country_name:
self.log(f"国家切换成功:{new_country}")
return True
else:
self.log(f"国家切换失败,当前国家:{new_country},目标国家:{country_name}")
return False
else:
self.log("无法验证切换结果")
return False
except Exception as e:
self.log(f"切换国家时发生异常:{traceback.format_exc()}")
return False
def need_login(self):
"""
判断是否需要登录,部分国家可能需要登录后才能切换国家
处理流程:
"""
time.sleep(3) # 等待页面可能的登录元素加载,避免跳转等等
self.tab.wait.doc_loaded(timeout=30, raise_err=False)
need_login_ele = self.tab.eles('xpath://h1[@class="a-spacing-small"]|//span[contains(text(),"登录")]', timeout=5)
if len(need_login_ele) > 0:
self.log("检测到需要登录元素")
return True
return False
def login(self, password, username=""):
try:
self.tab.wait.doc_loaded(timeout=30, raise_err=False)
for _ in range(4):
switch_account = self.tab.eles(
'xpath://div[@data-test-id="switchableAccounts"]//div[@class="a-fixed-left-grid"]', timeout=5)
if len(switch_account) > 0:
switch_account[0].click()
time.sleep(1)
self.tab.wait.doc_loaded(timeout=30, raise_err=False)
pwd_input = self.tab.eles('xpath://input[@type="password"]', timeout=5)
if len(pwd_input) > 0:
pwd_input[0].input(password, clear=True)
submit_btn = self.tab.eles('xpath://input[@id="signInSubmit"]', timeout=5)
if len(submit_btn) > 0:
submit_btn[0].click()
self.tab.wait.doc_loaded(timeout=30, raise_err=False)
send_code = self.tab.eles('xpath://span[@id="auth-send-code" and contains(string(.),"发送一次性密码")]',
timeout=5)
self.log(f"发送一次性密码:{len(send_code)}")
if len(send_code) > 0:
self.log("检测到 发送一次性密码")
send_code[0].click()
time.sleep(1)
self.tab.wait.doc_loaded(timeout=30, raise_err=False)
else:
for j in range(5):
opt_code_input = self.tab.eles('xpath://input[@name="otpCode"]', timeout=30)
self.log(f"验证码输入框:{len(opt_code_input)}")
if len(opt_code_input) > 0:
opt_code_input[0].wait.displayed(timeout=10, raise_err=False)
for _ in range(30):
if opt_code_input[0].value is not None and opt_code_input[0].value.strip() != "":
print("检测到验证码输入完成")
submit_btn = self.tab.eles('xpath://input[@id="auth-signin-button"]', timeout=10)
if len(submit_btn) > 0:
submit_btn[0].click()
self.tab.wait.doc_loaded(timeout=20, raise_err=False)
# return True
error_mes = self.tab.eles('xpath://div[@id="auth-error-message-box"]',
timeout=10)
if len(error_mes) > 0:
self.log("验证码输入错误")
self.log(f"验证码输入错误提示:{error_mes[0].text}")
self.tab.refresh()
else:
self.log("登入成功")
return True
time.sleep(1)
else:
break
submit_btn = self.tab.eles('xpath://input[@id="auth-signin-button"]', timeout=10)
if len(submit_btn) > 0:
submit_btn[0].click()
except Exception as e:
self.log(f"登录过程中发生异常:{traceback.format_exc()}")
return False
def SwitchPage(self):
"""
切换至 管理所有库存页面
1、等待 //navigation-favorites-bar[@class="hydrated"] 出现
"""
navigation = self.tab.ele('xpath://navigation-favorites-bar[@class="hydrated"]')
navigation.wait.displayed(raise_err=False)
page_btn = navigation.sr('xpath://internal-fav-bar-links[@data-internal="navigation"]').sr(
'xpath://a[@data-page-id="ezdpc-gui-inventory-mons"]')
page_btn.wait.displayed(raise_err=False)
page_btn.click(timeout=5)
self.tab.wait.doc_loaded()
# 等待搜索框出现
search_region = self.tab.ele('xpath://div[@id="searchBoxContainer"]//kat-input-group')
search_region.wait.displayed(raise_err=False, timeout=60)
def search(self,filter_type="ApprovalRequired"):
sku_ls = []
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]",timeout=5)
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
drop_down = self.tab.ele('xpath://div[contains(@class,"VolusListingStatusDropDown-module__verticalContainer")]//kat-dropdown')
drop_down.wait.displayed(raise_err=False)
drop_down.wait.enabled(raise_err=False)
time.sleep(0.6)
drop_down.click()
# //kat-option[@value="SearchSuppressed"]
xp = f'xpath://kat-option[@value="{filter_type}"]'
self.log(f"正在寻找筛选条件 {filter_type}xpath: {xp}")
approval_required = self.tab.eles(xp,timeout=5)
if len(approval_required) == 0:
self.log(f"没有需要{filter_type}选项】没有需要{filter_type}的商品了")
return sku_ls # "没有需要审批的商品了"
else:
approval_required = approval_required[0]
approval_required.wait.displayed(raise_err=False)
approval_required.click()
approval_required_text = approval_required.text
self.log(f"已选择筛选条件: {approval_required_text}")
count = re.findall(r'\d+', approval_required_text)
if count:
count = int(count[0])
self.log(f"待审批的商品数量: {count}")
if count <= 0:
self.log(f"没有需要{filter_type}的商品了")
return sku_ls #"没有需要审批的商品了"
for _ in range(3):
# 等待加载完成
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]")
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=3)
if len(sku_ls) > 0:
break
approval_required.click()
return sku_ls
class TaskBase:
task_name = "任务基类"
country_info = {
"DE": "德国",
"FR": "法国",
"ES": "西班牙",
"IT": "意大利",
"UK": "英国"
}
def __init__(self, user_info: dict = None):
"""初始化审批任务处理器
Args:
user_info: 用户信息字典,包含 company, username, password
"""
self.user_info = user_info or {}
self.running = True
def log(self, message: str, level: str = "INFO"):
"""日志输出
Args:
message: 日志消息
level: 日志级别
"""
try:
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if level == "ERROR":
show_notification(message, "error")
print(f"[{timestamp}] [{self.task_name}] [{level}] {message}")
except Exception as e:
print(f"输出出错,{e}")
def process_task(self, task_data: dict):
pass
def open_shop(self, cls: Type[T], max_retries: int, company_name: str, shop_name: str, iskill: bool = False) -> T:
error_info = ""
driver = None
for retry in range(max_retries):
try:
self.log(f"尝试打开店铺 {shop_name} (第 {retry + 1}/{max_retries} 次)")
if iskill:
self.log("重试前先杀掉浏览器进程...")
kill_process("v6")
kill_process("v5")
time.sleep(2)
# 组装用户信息并创建驱动
user_info = {
**self.user_info,
"company": company_name
}
driver = cls(user_info)
browser = driver.open_shop(shop_name)
if browser and browser != "店铺不存在":
self.log(f"成功打开店铺 {shop_name}")
else:
self.log(f"打开店铺失败: {browser}", "WARNING")
driver = None
continue
# 判断是否需要登录
need_login = driver.need_login()
print("【是否需要登录】:", need_login)
if need_login:
self.log(f"店铺 {shop_name} 需要登录,正在登录...")
# 获取店铺凭证
response = get_shop_info(shop_name)
print("【获取店铺凭证返回】:", response.text)
shop_data = response.json()
if not shop_data:
mes = f"获取店铺凭证失败,响应数据: {shop_data.get('message', '未知错误')}"
self.log(mes, "ERROR")
show_notification(mes, "ERROR")
continue
password = shop_data["data"]["password"]
login_success = driver.login(password)
if login_success:
self.log(f"店铺 {shop_name} 登录成功,正在重新打开店铺...")
browser = driver.open_shop(shop_name)
if browser and browser != "店铺不存在":
self.log(f"成功打开店铺 {shop_name} 登录后")
break
else:
self.log(f"登录后打开店铺失败: {browser}", "WARNING")
driver = None
else:
self.log(f"店铺 {shop_name} 登录失败", "WARNING")
driver = None
else:
break
except Exception as e:
import traceback
self.log(f"打开店铺异常: {traceback.format_exc()}", "INFO")
driver = None
error_info = str(e)
time.sleep(10)
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(3)
# 检查是否成功打开
if not driver or not browser or browser == "店铺不存在":
error_msg = f"店铺 {shop_name} 打开失败,已重试 {max_retries} 次,跳过该店铺,{error_info}"
self.log(error_msg, "ERROR")
# 从执行列表中移除
if shop_name in runing_shop:
del runing_shop[shop_name]
return driver
return driver
def process_shop(self, shop_data: Dict[str, Any], task_id: int):
pass
def action_init(self, driver, country_name, shop_name, risk_listing_filter=None):
"""切换到指定国际 -> 页面 -》 筛选好"""
max_retries = 5
switch_success = False
switch_success_pg = False
search_success = False
sku_ls = []
for retry in range(max_retries):
try:
self.log(f"尝试切换到国家 {country_name} (第 {retry + 1}/{max_retries} 次)")
if retry > 2:
# 刷新不行就重新打开店铺
self.log("重试前重新打开店铺...")
try:
driver.close_store()
time.sleep(3)
driver.open_shop(shop_name)
except Exception as e:
self.log(f"关闭重新打开店铺: {str(e)}", "WARNING")
# 如果不是第一次尝试,先刷新页面
if retry > 0:
self.log("重试前刷新页面...")
try:
driver.tab.refresh()
time.sleep(3)
except Exception as e:
self.log(f"刷新页面失败: {str(e)}", "WARNING")
switch_success = driver.SwitchingCountries(country_name)
if switch_success:
self.log(f"成功切换到国家 {country_name}")
# 切换到库存管理页面
driver.SwitchPage()
self.log(f"已切换到库存管理页面")
time.sleep(3)
switch_success_pg = True
if risk_listing_filter is None:
self.log(f"risk_listing_filter 为空,不筛选, {risk_listing_filter}")
search_success = True
break
sku_ls = driver.search(filter_type=risk_listing_filter)
search_success = True
self.log(f"国家 {country_name} 搜索出 {len(sku_ls)} 商品,开始处理...")
break
else:
self.log(f"切换到国家 {country_name} 失败", "WARNING")
except Exception as e:
import traceback
self.log(f"切换国家 {country_name} 异常: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(2)
return switch_success, switch_success_pg, search_success, sku_ls

View File

@@ -1,999 +0,0 @@
import time
import re
import traceback
from DrissionPage._pages.chromium_tab import ChromiumTab
from config import runing_task, runing_shop
from datetime import datetime
from amazon.del_brand import AmamzonBase, kill_process
# 导入 webview 用于前端通知
try:
import webview
except ImportError:
webview = None
class AmzoneApprove(AmamzonBase):
def SwitchPage(self):
"""
切换至 管理所有库存页面
1、等待 //navigation-favorites-bar[@class="hydrated"] 出现
"""
navigation = self.tab.ele('xpath://navigation-favorites-bar[@class="hydrated"]')
navigation.wait.displayed(raise_err=False)
page_btn = navigation.sr('xpath://internal-fav-bar-links[@data-internal="navigation"]').sr(
'xpath://a[@data-page-id="ezdpc-gui-inventory-mons"]')
page_btn.wait.displayed(raise_err=False)
page_btn.click(timeout=5)
self.tab.wait.doc_loaded()
# 等待搜索框出现
search_region = self.tab.ele('xpath://div[@id="searchBoxContainer"]//kat-input-group')
search_region.wait.displayed(raise_err=False)
def search(self,filter_type="ApprovalRequired"):
sku_ls = []
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]",timeout=5)
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
drop_down = self.tab.ele('xpath://div[contains(@class,"VolusListingStatusDropDown-module__verticalContainer")]//kat-dropdown')
drop_down.wait.displayed(raise_err=False)
drop_down.wait.enabled(raise_err=False)
time.sleep(0.6)
drop_down.click()
# //kat-option[@value="SearchSuppressed"]
xp = f'xpath://kat-option[@value="{filter_type}"]'
print(f"正在寻找筛选条件 {filter_type}xpath: {xp}")
approval_required = self.tab.eles(xp,timeout=5)
if len(approval_required) == 0:
print(f"【没有需要{filter_type}选项】没有需要{filter_type}的商品了")
return sku_ls # "没有需要审批的商品了"
else:
approval_required = approval_required[0]
approval_required.wait.displayed(raise_err=False)
approval_required.click()
approval_required_text = approval_required.text
print(f"已选择筛选条件: {approval_required_text}")
count = re.findall(r'\d+', approval_required_text)
if count:
count = int(count[0])
print(f"待审批的商品数量: {count}")
if count <= 0:
print(f"没有需要{filter_type}的商品了")
return sku_ls #"没有需要审批的商品了"
asin = "B0C7KT6ZZN"
search_input = self.tab.ele("xpath://kat-input[contains(@class,'SearchBox-module__searchInput')]").sr(
'xpath://span[@class="container"]//input[@part="input"]')
search_input.input(asin, clear=True)
search_btn = self.tab.ele("xpath://kat-icon[@name='search']")
search_btn.click()
for _ in range(3):
# 等待加载完成
load_ele = self.tab.ele("xpath://div[contains(@class,'Loader-module__loader')]")
load_ele.wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
search_input = self.tab.ele("xpath://kat-input[contains(@class,'SearchBox-module__searchInput')]").sr(
'xpath://span[@class="container"]//input[@part="input"]')
search_input.input(asin, clear=True)
search_btn = self.tab.ele("xpath://kat-icon[@name='search']")
search_btn.click()
load_ele = self.tab.ele("xpath://div[contains(@class,'Loader-module__loader')]")
load_ele.wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=3)
if len(sku_ls) > 0:
break
approval_required.click()
return sku_ls
def wait_loaded(self):
# 等待加载完成
try:
load_ele = self.tab.ele(
'xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]//div[contains(@class,"Loader-module__loader")]|//kat-panel[@data-testid="kat-panel-ActionPanelContent"]/div[@data-f1-component]//div[contains(@class,"==")]/div[contains(@class,"==")]/span',
timeout=3)
load_ele.wait.deleted(timeout=5, raise_err=False)
except Exception as e:
print("等待加载中消失出错", e)
def clear_tab(self):
all_tab = self.browser.get_tabs(title="亚马逊")
close_tab = []
for tab in all_tab:
tab_id = tab if isinstance(tab,str) else tab.tab_id
if self.tab.tab_id == tab:
continue
close_tab.append(tab)
print("需要关闭的标签页",close_tab)
print("当前操作的tab_id",self.tab.tab_id)
self.browser.close_tabs(close_tab)
def handle_repair_product(self,tab:ChromiumTab):
def wait_loaded():
# 等待加载完成
try:
load_ele = tab.ele(
'xpath://div[@class="contentWrapper"]//div[contains(@class,"==")]/span[not(node())]',
timeout=3)
load_ele.wait.deleted(timeout=5, raise_err=False)
except Exception as e:
print("等待加载中消失出错", e)
tab.wait.doc_loaded(timeout=30)
submit_btn_ls = tab.eles('xpath://div[@id="ahd-product-policies-table"]/div//div[@data-testid="nextStepsMetricWrapper"]//kat-button[@label="提交"]',timeout=10)
if len(submit_btn_ls) == 0:
print("未找到提交按钮,可能页面未加载完成或者页面结构发生变化")
return
# todo 遍历
for index in range(len(submit_btn_ls)):
if index > len(submit_btn_ls)-1:
print("提交按钮数量发生变化,停止处理")
break
submit_btn = submit_btn_ls[index]
# 关闭按钮
close_x_btn = tab.eles('xpath://div[@class="flyoutPanelContent"]/span',timeout=5)
if len(close_x_btn) > 0:
close_x_btn[0].wait.displayed(timeout=5, raise_err=False)
close_x_btn[0].click()
submit_btn.wait.enabled(timeout=5, raise_err=False)
submit_btn.click()
time.sleep(0.5)
self.wait_loaded()
wait_loaded()
need_input = tab.eles('xpath://div[@class="contentWrapper"]//kat-input[@data-testid="kat-input-dew:ump_epr_resgitration_number_title"]',timeout=5)
if len(need_input) > 0:
print("需要输入注册号,不符合操作要求,跳过...")
continue
target_title = tab.eles('xpath://div[@class="contentWrapper"]//section//h4[text()="警告和安全信息"]')
if len(target_title) > 0:
not_start = tab.eles('xpath://div[@class="contentWrapper"]//div[@aria-label="安全证明"]//kat-label[@text="未开始"]')
if len(not_start) == 0:
print("安全证明-不是未开始状态,不需要操作")
continue
not_start.wait.displayed(timeout=5, raise_err=False)
not_start.click()
wait_loaded()
check = tab.ele('xpath://div[@class="contentWrapper"]//kat-checkbox[@name="value"]')
check.wait.displayed(timeout=5, raise_err=False)
check.click()
save_btn = tab.ele('xpath://div[@class="contentWrapper"]//kat-button[@variant="primary"]')
save_btn.wait.enabled(timeout=5, raise_err=False)
save_btn.click()
wait_loaded()
close_btn = tab.ele('xpath://div[@class="contentWrapper"]//kat-button[@label="关闭"]',timeout=10)
close_btn.wait.displayed(timeout=5, raise_err=False)
close_btn.click()
save_btn_ls = tab.eles('xpath://div[@class="contentWrapper"]//kat-button[@variant="primary"]',timeout=5)
if len(save_btn_ls) > 0:
option_selection_ls = tab.eles('xpath://div[@class="contentWrapper"]//div[@role="option"]',timeout=5)
for option in option_selection_ls:
option.click()
time.sleep(0.5)
save_btn_ls[0].wait.enabled(timeout=5, raise_err=False)
save_btn_ls[0].click()
wait_loaded()
close_btn = tab.ele('xpath://div[@class="contentWrapper"]//kat-button[@label="关闭"]',timeout=10)
close_btn.wait.displayed(timeout=5, raise_err=False)
close_btn.wait.enabled(timeout=5, raise_err=False)
close_btn.click()
submit_btn_ls = tab.eles('xpath://div[@id="ahd-product-policies-table"]/div//div[@data-testid="nextStepsMetricWrapper"]//kat-button[@label="提交"]',timeout=10)
print("修复商品信息处理完成!")
tab.close()
def run_page_action(self):
print("开始执行")
num = 0
retry_num = 0
already_asin = set()
while retry_num < 3: # 最多重试3次
# if num > 3: #测试
# return
# 等待加载完成
try:
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]",timeout=5)
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
# 获取当前页码
try:
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
if len(page_pamel) > 0:
current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text
# 总页数
total_page = page_pamel[0].sr.eles('xpath:.//ul[@class="pages"]//span[@class="page__inner"][last()]')
if len(total_page) > 0:
total_page = total_page[-1].text
else:
total_page = 0
print(f"当前页码: {current_page} / 总页数: {total_page}")
except Exception as e:
print("获取页码失败", e)
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=10)
print(f"获取到 {len(sku_ls)}")
# for sku_ele in sku_ls[0:2]:
for sku_ele in sku_ls:
# solve_problem = sku_ele.eles('xpath:.//kat-link[@label="解决商品信息问题"]')
asin = sku_ele.ele('xpath:.//div[contains(@class,"JanusSplitBox-module__container")]//div[contains(@class,"JanusSplitBox-module__panel--") and contains(string(.),"ASIN")]/..//div[last()]').text
print(f"ASIN {asin} 存在问题,正在点击解决...")
if asin in already_asin:
print(f"{asin} 已经处理过了,跳过")
continue
solve_problem = sku_ele.eles('xpath:.//kat-link[@label="解决商品信息问题"]|.//kat-link[@label="修复被禁止显示的商品"]|.//kat-link[@label="解决商品移除风险"]')
if len(solve_problem) == 0:
print(f"{asin},没有处理入口,不处理")
yield (asin,"没有处理入口,不处理")
already_asin.add(asin)
continue
solve_problem[0].click()
action_panel = self.tab.ele('xpath://kat-panel-wrapper[@data-testid="kat-panel-wrapper-ActionPanelContent"]')
action_panel.wait.displayed(timeout=5, raise_err=False)
self.wait_loaded()
try:
# 如果存在“请求批准”按钮,则直接返回跳过
request_approval_btn = self.tab.eles('xpath://kat-panel-wrapper[@data-testid="kat-panel-wrapper-ActionPanelContent"]//div[@data-testid="section-header" and contains(string(.),"移除")]',timeout=5)
if len(request_approval_btn) > 0:
print(f"ASIN {asin} 存在请求批准按钮,不需要处理,跳过...")
panel_close_btn = self.tab.ele('xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]',
timeout=5).sr('xpath:.//button[@class="close"]')
panel_close_btn.click()
action_panel.wait.deleted(timeout=3, raise_err=False)
yield (asin,"请求批准")
# 无需采取任何操作
not_operate = self.tab.eles('xpath://kat-alert[contains(@description,"如果您之前已提交更改,则这些更改当前正在处理中") and not(@dismissed)]',timeout=3)
if len(not_operate) > 0:
print(f"ASIN {asin} 无需采取任何操作,跳过...")
yield (asin,"无需操作")
already_asin.add(asin)
continue
# 解决商品信息违规问题 按钮
kat_box_ls = action_panel.eles('xpath:.//kat-box[@variant="white"]',timeout=5)
# if len(kat_box_ls) == 0:
# print(f"ASIN {asin} 的操作面板中未找到选项,跳过...")
# continue
print(f"ASIN {asin}需要处理的有:{len(kat_box_ls)}个问题.")
# 解决商品信息违规问题 按钮,有多少个都要处理
for i in range(len(kat_box_ls)):
print(f"开始处理第{i}个问题")
if len(kat_box_ls) <= i:
print(f"ASIN {asin} 的操作面板中选项数量发生变化,停止处理...")
break
kat_box_ls[i].click()
time.sleep(0.5)
self.wait_loaded()
# TODO 增加 “出现警告和安全信息时候下滑” 的情况
safe_handle_ls = self.tab.eles('xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]//div[@aria-label="安全证明"]//kat-label[@text="未开始"]',
timeout=5)
if len(safe_handle_ls) > 0:
print("存在安全证明,正在处理...")
safe_handle_ls[0].click()
time.sleep(0.5)
self.wait_loaded()
check_box = self.tab.ele('xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]//kat-checkbox')
check_box.click()
else:
problem_selection_ls = self.tab.eles(
'xpath://kat-panel-wrapper[@data-testid="kat-panel-wrapper-ActionPanelContent"]//div[@data-testid="registry-list"]//div[@data-testid="registry"]',
timeout=10)
print("待选择数量",len(problem_selection_ls))
for problem_selection in problem_selection_ls:
problem_selection.click()
print("点击选择完成")
time.sleep(0.5)
# 点击保存
confirm_btn = self.tab.ele('xpath://kat-panel-wrapper[@data-testid="kat-panel-wrapper-ActionPanelContent"]//kat-button[@variant="primary"]')
confirm_btn.wait.displayed(timeout=3, raise_err=False)
confirm_btn.click()
self.wait_loaded()
# 等待关闭按钮出现
try:
close_btn = self.tab.ele('xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]//kat-button[@label="关闭"]',
timeout=10)
close_btn.click()
except Exception as e:
print("点击关闭按钮失败",e)
kat_box_ls = action_panel.eles('xpath:.//kat-box[@variant="white"]', timeout=5)
# 修复商品信息
repair_product = action_panel.eles('xpath:.//kat-button[@class="action-button"]',timeout=5)
if len(repair_product) > 0:
print(f"ASIN {asin} 存在修复商品信息按钮,正在点击...")
self.clear_tab()
repair_product[0].click()
time.sleep(1)
# 获取最新的tab
new_tab = self.browser.latest_tab
if isinstance(new_tab, str):
new_tab = self.browser.get_tab(id_or_num=new_tab)
self.handle_repair_product(new_tab)
except Exception as e:
print("【asin】",asin,"处理失败",traceback.format_exc())
yield (asin,"处理失败")
already_asin.add(asin)
continue
yield (asin,"处理完成")
already_asin.add(asin)
try:
# 全部操作完成,关闭
panel_close_btn= self.tab.ele('xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]',
timeout=5).sr('xpath:.//button[@class="close"]')
panel_close_btn.click()
action_panel.wait.deleted(timeout=3, raise_err=False)
except Exception as e:
print("关闭操作面板失败", e)
# 判断是否存在需要翻页的情况
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
if len(page_pamel) == 0:
break
next_page_btn = page_pamel[0].sr('xpath:.//span[@part="pagination-nav-right"]')
class_str = next_page_btn.attr('class')
if "end" in class_str:
break
next_page_btn.click()
num += 1
print(f"【程序计算】正在翻页,已翻 {num} 页...")
already_asin = set()
except Exception as e:
print("处理审批操作异常", e)
traceback.print_exc()
retry_num += 1
self.tab.refresh()
self.tab.wait.doc_loaded(raise_err=False,timeout=120)
class ApproveTask:
"""审批任务处理类:负责处理产品风险审批任务"""
country_info = {
"DE": "德国",
"FR": "法国",
"ES": "西班牙",
"IT": "意大利",
"UK": "英国"
}
@staticmethod
def show_notification(message: str, message_type: str = "error"):
"""显示 pywebview 顶层通知5秒后自动消失
Args:
message: 通知消息
message_type: 消息类型 (success/warning/error/info)
"""
if webview and webview.windows:
try:
# 转义单引号,防止 JavaScript 语法错误
safe_message = message.replace("'", "\\'").replace('"', '\\"').replace('\n', '\\n')
# 设置通知样式颜色
color_map = {
'success': '#67C23A',
'warning': '#E6A23C',
'error': '#F56C6C',
'info': '#909399'
}
bg_color = color_map.get(message_type, '#F56C6C')
# 构建前端通知的 JavaScript 代码 - 创建原生 HTML 通知
js_code = f"""
(function() {{
// 移除已存在的通知
var existingNotif = document.getElementById('pywebview-notification');
if (existingNotif) {{
existingNotif.remove();
}}
// 创建通知容器
var notif = document.createElement('div');
notif.id = 'pywebview-notification';
notif.style.cssText = 'position: fixed; top: 20px; left: 50%; transform: translateX(-50%); ' +
'background: {bg_color}; color: white; padding: 12px 20px; border-radius: 4px; ' +
'box-shadow: 0 2px 12px rgba(0,0,0,0.3); z-index: 99999; font-size: 14px; ' +
'max-width: 600px; word-wrap: break-word; display: flex; align-items: center; gap: 10px;';
// 添加消息内容
var msgSpan = document.createElement('span');
msgSpan.textContent = '{safe_message}';
notif.appendChild(msgSpan);
// 添加关闭按钮
var closeBtn = document.createElement('span');
closeBtn.innerHTML = '×';
closeBtn.style.cssText = 'cursor: pointer; font-size: 18px; font-weight: bold; margin-left: 10px;';
closeBtn.onclick = function() {{ notif.remove(); }};
notif.appendChild(closeBtn);
// 添加到页面
document.body.appendChild(notif);
// 5秒后自动消失
setTimeout(function() {{
if (notif && notif.parentNode) {{
notif.style.transition = 'opacity 0.3s';
notif.style.opacity = '0';
setTimeout(function() {{ notif.remove(); }}, 300);
}}
}}, 5000);
}})();
"""
# 在第一个窗口中执行 JavaScript
webview.windows[0].evaluate_js(js_code)
except Exception as e:
print(f"显示通知失败: {str(e)}")
"""
处理流程:
1、解析任务json,提取对应的companyName 组组装user_info实例化 AmzoneApprove
2、打开对应的店铺、遍历所有的国家站点处理
3、切换至指定国家切换至 管理所有库存页面
4、调用search方法查询出是否存在需要审批的商品如果没有则结束如果有则进入处理流程
5、进入处理流程后调用run_page_action方法(注意是通过yield 返回数据的)
任务json示例
{
"type": "product-risk-resolve-run",
"ts": 1775395996876,
"data": {
"taskId": 925,
"items": [
{
"matched": true,
"platform": "亚马逊",
"shopName": "魏振峰",
"shopId": "27543917795757",
"companyName": "rongchuang123",
"openStoreUrl": null,
"matchedUserId": 17543915345493,
"matchStatus": "MATCHED",
"matchMessage": null
}
],
"country_codes": [
"UK",
"DE",
"FR",
"ES"
]
}
}
"""
def __init__(self, user_info: dict = None):
"""初始化审批任务处理器
Args:
user_info: 用户信息字典,包含 company, username, password
"""
self.user_info = user_info or {}
self.running = True
def log(self, message: str, level: str = "INFO"):
"""日志输出
Args:
message: 日志消息
level: 日志级别
"""
from datetime import datetime
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if level == "ERROR":
self.show_notification(message, "error")
print(f"[{timestamp}] [ApproveTask] [{level}] {message}")
def process_task(self, task_data: dict):
"""处理审批任务主入口
Args:
task_data: 任务数据
"""
try:
data = task_data.get("data", {})
task_id = data.get("taskId")
items = data.get("items", [])
country_codes = data.get("country_codes", [])
risk_listing_filter = data.get("risk_listing_filter", "")
if not task_id:
self.log("任务ID为空跳过", "WARNING")
return
if not items:
self.log("店铺列表为空,跳过", "WARNING")
return
if not country_codes:
self.log("国家列表为空,跳过", "WARNING")
return
self.log(f"开始处理审批任务 {task_id},共 {len(items)} 个店铺,{len(country_codes)} 个国家")
from config import runing_task
runing_task[task_id] = {
"status": "running",
"start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"total_shops": len(items),
"processed_shops": 0,
"total_countries": len(country_codes) * len(items),
"processed_countries": 0,
"total_asins": 0,
"processed_asins": 0,
"success_count": 0,
"failed_count": 0,
"stop_requested": False
}
# 遍历处理每个店铺
for idx, shop_item in enumerate(items, 1):
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理", "WARNING")
runing_task[task_id]["status"] = "stopped"
return
shop_name = shop_item.get("shopName", "未知店铺")
self.log(f"[{idx}/{len(items)}] 开始处理店铺: {shop_name}")
self.show_notification(f"开始处理店铺: {shop_name}", "info")
try:
self.process_shop(shop_item, country_codes, task_id,risk_listing_filter)
# 更新已处理店铺数
if task_id in runing_task:
runing_task[task_id]["processed_shops"] += 1
except Exception as e:
import traceback
self.log(f"处理店铺 {shop_name} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 更新任务状态
if task_id in runing_task:
if runing_task[task_id].get("stop_requested", False):
runing_task[task_id]["status"] = "stopped"
self.log(f"任务 {task_id} 已被暂停!")
else:
runing_task[task_id]["status"] = "completed"
self.log(f"任务 {task_id} 处理完成!")
except Exception as e:
import traceback
self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR")
if task_id:
from config import runing_task
if task_id in runing_task:
runing_task[task_id]["status"] = "failed"
runing_task[task_id]["error"] = str(e)
def process_shop(self, shop_item: dict, country_codes: list, task_id: int, risk_listing_filter: str):
"""处理单个店铺
Args:
shop_item: 店铺信息
country_codes: 国家代码列表
task_id: 任务ID
risk_listing_filter: 风险商品筛选条件
"""
shop_name = shop_item.get("shopName", "未知店铺")
company_name = shop_item.get("companyName", "")
if not company_name:
self.log(f"店铺 {shop_name} 的公司名称为空,跳过", "WARNING")
return
if task_id in runing_task:
runing_task[task_id]["current_shop"] = shop_name
# 将店铺添加到正在执行中的店铺列表
start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
runing_shop[shop_name] = start_time
self.log(f"店铺 {shop_name} 已添加到执行列表,账号: {company_name},开始时间: {start_time}")
# 店铺打开重试最多3次
driver = None
max_retries = 3
error_info = ""
for retry in range(max_retries):
try:
self.log(f"尝试打开店铺 {shop_name} (第 {retry + 1}/{max_retries} 次)")
# 如果不是第一次尝试,先杀进程
# if retry > 0:
# self.log("重试前先杀掉浏览器进程...")
# kill_process("v6")
# kill_process("v5")
# time.sleep(2)
# 组装用户信息并创建驱动
user_info = {
**self.user_info,
"company": company_name
}
driver = AmzoneApprove(user_info)
browser = driver.open_shop(shop_name)
if browser and browser != "店铺不存在":
self.log(f"成功打开店铺 {shop_name}")
break
else:
self.log(f"打开店铺失败: {browser}", "WARNING")
driver = None
# 判断是否需要登录
need_login = driver.need_login()
if need_login:
self.log(f"店铺 {shop_name} 需要登录,正在登录...")
password = ""
login_success = driver.login(password)
if login_success:
self.log(f"店铺 {shop_name} 登录成功,正在重新打开店铺...")
browser = driver.open_shop(shop_name)
if browser and browser != "店铺不存在":
self.log(f"成功打开店铺 {shop_name} 登录后")
break
else:
self.log(f"登录后打开店铺失败: {browser}", "WARNING")
driver = None
else:
self.log(f"店铺 {shop_name} 登录失败", "WARNING")
driver = None
except Exception as e:
import traceback
self.log(f"打开店铺异常: {traceback.format_exc()}", "INFO")
driver = None
error_info = str(e)
time.sleep(10)
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(3)
# 检查是否成功打开
if not driver or not browser or browser == "店铺不存在":
error_msg = f"店铺 {shop_name} 打开失败,已重试 {max_retries} 次,跳过该店铺,{error_info}"
self.log(error_msg, "ERROR")
# 从执行列表中移除
if shop_name in runing_shop:
del runing_shop[shop_name]
return
try:
# 处理每个国家
for country_code in country_codes:
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理国家", "WARNING")
break
try:
self.process_country(driver, country_code, task_id, shop_name,risk_listing_filter)
except Exception as e:
import traceback
self.log(f"处理国家 {country_code} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 最后回传,标记完成
try:
self.post_result(task_id, shop_name, country_code, "", "", is_done=True)
except Exception as e:
self.log(f"回传结果失败: {str(e)}", "ERROR")
finally:
# 关闭店铺
try:
if driver:
self.log(f"关闭店铺 {shop_name}")
driver.close_store()
time.sleep(2)
except Exception as e:
self.log(f"关闭店铺失败: {str(e)}", "WARNING")
# 从正在执行中的店铺列表中移除
if shop_name in runing_shop:
del runing_shop[shop_name]
self.log(f"店铺 {shop_name} 已从执行列表中移除")
def process_country(self, driver: AmzoneApprove, country_code: str, task_id: int, shop_name: str, risk_listing_filter: str):
"""处理单个国家的审批任务
Args:
driver: AmzoneApprove驱动实例
country_code: 国家代码(如 UK, DE, FR 等)
task_id: 任务ID
shop_name: 店铺名称
risk_listing_filter: 风险商品筛选条件
"""
from config import runing_task
# 转换国家代码为中文名称
country_name = self.country_info.get(country_code, country_code)
info_mes = f"开始处理国家: {country_name} ({country_code})"
self.log(info_mes)
self.show_notification(info_mes, "info")
# 更新当前处理的国家
if task_id in runing_task:
runing_task[task_id]["current_country"] = country_name
# 切换国家最多重试3次
max_retries = 3
switch_success = False
for retry in range(max_retries):
try:
self.log(f"尝试切换到国家 {country_name} (第 {retry + 1}/{max_retries} 次)")
# 如果不是第一次尝试,先刷新页面
if retry > 0:
self.log("重试前刷新页面...")
try:
driver.tab.refresh()
time.sleep(3)
except Exception as e:
self.log(f"刷新页面失败: {str(e)}", "WARNING")
switch_success = driver.SwitchingCountries(country_name)
if switch_success:
self.log(f"成功切换到国家 {country_name}")
break
else:
self.log(f"切换到国家 {country_name} 失败", "WARNING")
except Exception as e:
import traceback
self.log(f"切换国家 {country_name} 异常: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(2)
# 如果切换失败,直接返回
if not switch_success:
error_message = f"切换到国家 {country_name} 失败,已重试 {max_retries} 次,跳过该国家"
self.log(error_message, "ERROR")
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
return
# 切换到库存管理页面
try:
driver.SwitchPage()
self.log(f"已切换到库存管理页面")
except Exception as e:
import traceback
self.log(f"切换页面失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
return
# 搜索需要审批的商品最多重试3次
sku_ls = []
for retry in range(max_retries):
try:
self.log(f"尝试搜索需要审批的商品 (第 {retry + 1}/{max_retries} 次)")
sku_ls = driver.search(filter_type=risk_listing_filter)
break
except Exception as e:
self.log(f"搜索商品异常: {str(e)}", "ERROR")
if retry < max_retries - 1:
try:
driver.tab.refresh()
time.sleep(3)
except Exception as refresh_error:
self.log(f"刷新页面失败: {str(refresh_error)}", "WARNING")
# 如果没有需要审批的商品,直接返回
if len(sku_ls) == 0:
self.log(f"国家 {country_name} 没有需要审批的商品")
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
return
self.log(f"国家 {country_name}{len(sku_ls)} 个需要审批的商品,开始处理...")
# 处理所有需要审批的商品通过yield获取结果
try:
for asin, status in driver.run_page_action():
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求停止处理ASIN", "WARNING")
break
self.log(f"ASIN {asin} 处理结果: {status}")
# 更新任务状态
if task_id in runing_task:
runing_task[task_id]["current_asin"] = asin
runing_task[task_id]["processed_asins"] += 1
if status == "处理完成":
runing_task[task_id]["success_count"] += 1
elif status == "请求批准":
# 请求批准的商品也算作成功(因为不需要处理)
runing_task[task_id]["success_count"] += 1
else:
runing_task[task_id]["failed_count"] += 1
# 回传结果到API
try:
self.post_result(task_id, shop_name, country_code, asin, status)
except Exception as e:
self.log(f"回传结果失败: {str(e)}", "ERROR")
except Exception as e:
import traceback
self.log(f"处理审批商品异常: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 更新已处理国家数
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
self.log(f"国家 {country_name} 处理完成")
def post_result(self, task_id: int, shop_name: str, country_code: str, asin: str, status: str,is_done: bool = False):
"""回传处理结果到API
Args:
task_id: 任务ID
shop_name: 店铺名称
country_code: 国家代码
asin: ASIN
status: 处理状态
"""
import requests
from config import DELETE_BRAND_API_BASE
url = f"{DELETE_BRAND_API_BASE}/api/product-risk-resolve/tasks/{task_id}/result"
if status == "请求批准":
country_data = {
"status": "",
"shopName": shop_name,
"productAsinSku": "",
"done": is_done,
"removeAsin": asin,
"removeStatus": status
}
else:
country_data = {
"status": status,
"shopName": shop_name,
"productAsinSku": asin,
"done": is_done,
"removeAsin": "",
"removeStatus": ""
}
payload = {
"shops": [
{
"error": "",
"countries": {
country_code : [ country_data ]
},
"shopName": shop_name
}
]
}
max_retries = 3
for retry in range(max_retries):
try:
print("=====================================")
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
self.log(f"回传URL: {url}")
self.log(f"回传数据: {payload}")
response = requests.post(
url,
json=payload,
headers={"Content-Type": "application/json"},
timeout=30,
verify=False
)
self.log(f"回传结果: {response.text}")
if response.status_code == 200:
self.log(f"结果回传成功: {asin} - {status}")
return
else:
self.log(f"结果回传失败,状态码: {response.status_code}", "WARNING")
print("=====================================")
except Exception as e:
self.log(f"调用API异常: {str(e)}", "ERROR")
print("=====================================")
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(2)
self.log(f"已达到最大重试次数,结果回传最终失败", "ERROR")
if __name__ == '__main__':
user_info = {
"company": "rongchuang123",
"username": "自动化_Robot",
"password": "#20zsg25"
}
shop_name = "魏振峰"
country = "西班牙"
kill_process('v6')
driver = AmzoneApprove(user_info)
browser = driver.open_shop(shop_name)
sw_suc = driver.SwitchingCountries(country)
driver.SwitchPage()
risk_listing_filter = "Active"
for _ in range(3):
try:
sku_ls = driver.search(filter_type=risk_listing_filter)
break
except Exception as e:
print(e)
driver.tab.refresh()
if len(sku_ls) > 0:
print("有数据,开始操作")
for asin, status in driver.run_page_action():
print(f"ASIN {asin} 的处理结果: {status}")
print("已完成操作")
# for i in range(0):
# print(i)

View File

@@ -1,948 +0,0 @@
import time
import re
import traceback
import requests
from DrissionPage._pages.chromium_tab import ChromiumTab
from config import runing_task, runing_shop,DELETE_BRAND_API_BASE
from datetime import datetime
from amazon.amazon_base import AmamzonBase,kill_process,TaskBase
from amazon.tool import show_notification
class AmzoneApprove(AmamzonBase):
mark_name = "产品风险审批"
def __init__(self, user_info: dict, socket_port: int = 19890):
super().__init__(user_info,socket_port)
self.already_asin = set()
def reset_already_asin(self):
self.already_asin = set()
self.log("already_asin 已重置")
def wait_loaded(self):
# 等待加载完成
try:
load_ele = self.tab.eles(
'xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]//div[contains(@class,"Loader-module__loader")]|//kat-panel[@data-testid="kat-panel-ActionPanelContent"]/div[@data-f1-component]//div[contains(@class,"==")]/div[contains(@class,"==")]/span',
timeout=3)
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=5, raise_err=False)
except Exception as e:
print("等待加载中消失出错", e)
def clear_tab(self):
for i in ["亚马逊","您的账户是否符合条件?"]:
all_tab = self.browser.get_tabs(title=i)
close_tab = []
for tab in all_tab:
tab_id = tab if isinstance(tab,str) else tab.tab_id
if self.tab.tab_id == tab_id:
continue
close_tab.append(tab)
print("需要关闭的标签页",close_tab)
print("当前操作的tab_id",self.tab.tab_id)
self.browser.close_tabs(close_tab)
def handle_repair_product(self,tab:ChromiumTab,headle_type=""):
def wait_loaded(wait_type="contentWrapper"):
# 等待加载完成
print("------------【修复商品信息】等待加载完成-----------------")
for _ in range(3):
try:
load_ele = tab.eles(
'xpath://div[@class="contentWrapper"]//div[contains(@class,"==")]/span[not(node())]|//kat-spinner[@id="ahd-app-page-spinner"]',
timeout=10)
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=20, raise_err=False)
if wait_type == "contentWrapper":
contentWrapper = tab.eles('xpath://div[@class="contentWrapper"]//h4',timeout=20)
if len(contentWrapper) > 0:
contentWrapper[0].wait.displayed(timeout=20, raise_err=True)
break
else:
time.sleep(2)
break
if wait_type == "content":
contentWrapper = tab.eles('xpath://div[@class="contentWrapper"]',timeout=20)
if len(contentWrapper) > 0:
contentWrapper[0].wait.displayed(timeout=20, raise_err=True)
break
else:
time.sleep(2)
break
if wait_type == "section":
section = tab.eles('xpath://section[@class="ahd-product-policy__section"]',timeout=20)
if len(section) > 0:
section[0].wait.displayed(timeout=20, raise_err=True)
break
except Exception as e:
print("等待加载中消失出错", e)
print("-----------------------------")
def close_contentWrapper():
# 关闭按钮
close_x_btn = tab.eles('xpath://div[@class="flyoutPanelContent"]/span',timeout=5)
contentWrapper = tab.eles('xpath://div[@class="flyoutPanelContent"]/div[@class="contentWrapper"]',timeout=5)
if len(close_x_btn) > 0 and len(contentWrapper)>0 :
if len(contentWrapper[0].children()) > 0:
close_x_btn[0].wait.displayed(timeout=5, raise_err=False)
close_x_btn[0].click()
tab.wait.doc_loaded(timeout=60,raise_err=False)
wait_loaded(wait_type="section")
# product_policy = tab.eles('xpath://div[@data-testid="ahd-desktop-product-policy"]')
product_policy = tab.eles('xpath://div[@data-testid="ahd-desktop-product-policy" and @class="ahd-product-policy-table-row-wrapper"]|//div[@class="ahd-product-policy-table-row-wrapper-nested" and @data-testid="ahd-desktop-product-policy"]')
if len(product_policy) == 0:
print("未找到提交处理区域,可能页面未加载完成或者页面结构发生变化")
time.sleep(5)
# product_policy = tab.eles('xpath://div[@data-testid="ahd-desktop-product-policy"]')
product_policy = tab.eles('xpath://div[@data-testid="ahd-desktop-product-policy" and @class="ahd-product-policy-table-row-wrapper"]|//div[@class="ahd-product-policy-table-row-wrapper-nested" and @data-testid="ahd-desktop-product-policy"]')
print("等待5秒后再次查找提交处理区域",len(product_policy))
# submit_btn_ls = tab.eles('xpath://div[@id="ahd-product-policies-table"]/div//div[@data-testid="nextStepsMetricWrapper"]//kat-button[@label="提交"]',timeout=10)
# if len(submit_btn_ls) == 0:
# return
now_index = 0
submit_index = 0
for _ in range(3):
try:
# 先全部点开
chevron_right_ls = tab.eles('xpath://kat-icon[@name="chevron-right"]',timeout=10)
for ele in chevron_right_ls:
ele.click()
time.sleep(0.5)
except Exception as e:
print("【修复商品信息】全部点开失败",e)
try:
# todo 遍历
print("【商品修复产品卡片】数量",len(product_policy))
print("【商品修复产品卡片】列表",product_policy)
for index in range(now_index,len(product_policy)):
product_policy = tab.eles('xpath://div[@data-testid="ahd-desktop-product-policy" and @class="ahd-product-policy-table-row-wrapper"]|//div[@class="ahd-product-policy-table-row-wrapper-nested" and @data-testid="ahd-desktop-product-policy"]')
now_index = index
if index > len(product_policy)-1:
print("提交按钮区域发生变化,停止处理")
break
message_link = product_policy[index].eles('xpath:.//a[@data-testid="ahd-reason-message-link"]')
message = message_link[0].text
if headle_type in ['查看冻结原因']:
need_heanle = any(i in message for i in ["制造商详细联系信息","警告和安全信息","负责人详细联系信息"])
else:
need_heanle = True
if not need_heanle:
print(f"{headle_type}】【{message}】不需要处理,跳过")
continue
spread_btn = product_policy[index].eles('xpath:.//kat-icon[@name="chevron-right"]',timeout=5)
print(f"{index},是否存在展开按钮",len(spread_btn))
if len(spread_btn) > 0:
spread_btn[0].click()
time.sleep(0.5)
submit_btn_ls = product_policy[index].eles('xpath:.//kat-button[@label="提交"]',timeout=5)
# try:
# submit_btn = product_policy[index].ele('xpath:.//kat-button[@label="提交"]',timeout=5)
# except Exception as e:
# print(datetime.now().strftime("%Y-%m-%d %H:%M:%S") ,"没有查找到提交按钮,跳过!")
# close_contentWrapper()
# continue
# print("提交按钮数量:",len(submit_btn))
print("提交按钮列表:",submit_btn_ls)
if len(submit_btn_ls) > 0:
print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"),"查找到提交按钮")
# submit_btn = submit_btn[0]
submit_btn = tab.eles('xpath://kat-button[@label="提交"]',timeout=10)[submit_index]
submit_index += 1
else:
print(datetime.now().strftime("%Y-%m-%d %H:%M:%S") ,"没有查找到提交按钮,跳过!")
close_contentWrapper()
continue
# time.sleep(10)
# 关闭按钮
close_x_btn = tab.eles('xpath://div[@class="flyoutPanelContent"]/span',timeout=5)
contentWrapper = tab.eles('xpath://div[@class="flyoutPanelContent"]/div[@class="contentWrapper"]',timeout=5)
if len(close_x_btn) > 0 and len(contentWrapper)>0 :
if len(contentWrapper[0].children()) > 0:
close_x_btn[0].wait.displayed(timeout=5, raise_err=False)
close_x_btn[0].click()
print("开始操作",submit_btn)
print("操作的ID 为:",submit_btn.attr('id'))
submit_btn.wait.enabled(timeout=5, raise_err=False)
submit_btn.click()
# 先直接等待contentWrapper出现不要进行复杂的元素查找
try:
# 使用简单的等待,避免触发页面事件
for check_count in range(10):
contentWrapper_exists = tab.eles('xpath://div[@class="contentWrapper"]', timeout=2)
if len(contentWrapper_exists) > 0:
print(f"【弹窗已出现】第{check_count+1}次检查")
time.sleep(2) # 再等待2秒让内容完全加载
break
time.sleep(1)
except Exception as e:
print(f"等待弹窗出现异常: {e}")
# wait_loaded(wait_type="content")
for j in range(4):
if j == 0:
timeout = 5
else:
timeout = 10
contentWrapper = tab.eles('xpath://div[@class="contentWrapper"]//h4',timeout=timeout)
print("标题的数量",len(contentWrapper))
if len(contentWrapper) > 0:
contentWrapper[0].wait.displayed(timeout=20, raise_err=True)
break
else:
# 关闭按钮
close_contentWrapper()
time.sleep(2)
submit_btn.click()
need_input = tab.eles('xpath://div[@class="contentWrapper"]//kat-input[@data-testid="kat-input-dew:ump_epr_resgitration_number_title"]',timeout=5)
if len(need_input) > 0:
print("需要输入注册号,不符合操作要求,跳过...")
close_contentWrapper()
continue
target_title = tab.eles('xpath://div[@class="contentWrapper"]//section//h4[text()="警告和安全信息"]',timeout=10)
print(f"序号{index}【警告和安全信息】数量 ->",len(target_title))
if len(target_title) > 0:
is_Continue = False
for j in range(10):
not_start = tab.eles('xpath://div[@class="contentWrapper"]//div[@aria-label="安全证明"]//kat-label',timeout=2)
if len(not_start) > 0:
status_lebel = not_start[0].attr("text")
print(f"序号{index}【警告和安全信息】状态 ->",status_lebel)
if status_lebel and str(status_lebel).strip() != "" and status_lebel != "未开始":
print("安全证明-不是未开始状态,不需要操作")
close_contentWrapper()
continue
if status_lebel == "未开始":
is_Continue = True
break
else:
time.sleep(1)
# not_start = tab.eles('xpath://div[@class="contentWrapper"]//div[@aria-label="安全证明"]//kat-label[@text="未开始"]',timeout=10)
# if len(not_start) == 0:
# print("安全证明-不是未开始状态,不需要操作")
# close_contentWrapper()
# continue
print(f"序号{index}【警告和安全信息】是否需要继续 ->",is_Continue)
if not is_Continue:
continue
not_start = not_start[0]
not_start.wait.displayed(timeout=5, raise_err=False)
not_start.click()
wait_loaded()
check = tab.ele('xpath://div[@class="contentWrapper"]//kat-checkbox[@name="value"]',timeout=10)
check.wait.displayed(timeout=5, raise_err=False)
check.click()
save_btn = tab.ele('xpath://div[@class="contentWrapper"]//kat-button[@variant="primary"]',timeout=10)
save_btn.wait.enabled(timeout=5, raise_err=False)
save_btn.click()
wait_loaded()
close_btn = tab.ele('xpath://div[@class="contentWrapper"]//kat-button[@label="关闭"]',timeout=10)
close_btn.wait.displayed(timeout=5, raise_err=False)
close_btn.click()
print(f"序号{index}【警告和安全信息】操作完成")
continue
save_btn_ls = tab.eles('xpath://div[@class="contentWrapper"]//kat-button[@variant="primary"]',timeout=10)
print(f"序号{index}【选择保存】是否存在->",len(save_btn_ls))
if len(save_btn_ls) > 0:
option_selection_ls = tab.eles('xpath://div[@class="contentWrapper"]//div[@role="option"]',timeout=5)
for option in option_selection_ls:
option.click()
time.sleep(0.5)
save_btn_ls[0].wait.enabled(timeout=5, raise_err=False)
save_btn_ls[0].click()
wait_loaded()
close_contentWrapper()
print(f"序号{index}【选择保存】操作完成")
continue
submit_btn_ls = tab.eles('xpath://div[@id="ahd-product-policies-table"]/div//div[@data-testid="nextStepsMetricWrapper"]//kat-button[@label="提交"]',timeout=10)
if len(submit_btn_ls) > 0:
submit_btn_ls[0].click()
submit_btn_ls[0].wait.deleted(timeout=5, raise_err=False)
wait_loaded()
close_contentWrapper()
break
except Exception as e:
print(f"【handle_repair_product】处理修复商品信息异常", traceback.format_exc())
tab.refresh()
tab.wait.doc_loaded(timeout=60,raise_err=False)
time.sleep(2)
wait_loaded(wait_type="section")
# product_policy = tab.eles('xpath://div[@data-testid="ahd-desktop-product-policy"]')
product_policy = tab.eles('xpath://div[@data-testid="ahd-desktop-product-policy" and @class="ahd-product-policy-table-row-wrapper"]|//div[@class="ahd-product-policy-table-row-wrapper-nested" and @data-testid="ahd-desktop-product-policy"]')
print("修复商品信息处理完成!")
tab.close()
def run_page_action(self):
self.log(f"==================={self.mark_name}=======================")
self.log("开始执行...")
num = 0
retry_num = 0
max_retry_num = 5
while retry_num < max_retry_num: # 最多重试3次
# if num > 3: #测试
# return
# 等待加载完成
try:
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]",timeout=5)
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
# 获取当前页码
try:
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
if len(page_pamel) > 0:
current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text
# 总页数
total_page = page_pamel[0].sr.eles('xpath:.//ul[@class="pages"]//span[@class="page__inner"][last()]')
if len(total_page) > 0:
total_page = total_page[-1].text
else:
total_page = 0
self.log(f"当前页码: {current_page} / 总页数: {total_page}")
except Exception as e:
self.log(f"获取页码失败:{e}")
# 保存当前URL用于失败重试时访问
try:
self.url = self.tab.url
self.log(f"获取当前的链接:{self.url}")
except Exception as e:
self.url = None
self.log(f"获取当前的链接失败:{e}")
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=10)
self.log(f"获取到 {len(sku_ls)}")
if len(sku_ls) == 0:
retry_num += 1
print(f"没有获取到 ASIN 商品,重试{retry_num}/{max_retry_num}")
self.tab.refresh()
self.tab.wait.doc_loaded(raise_err=False, timeout=120)
if retry_num == max_retry_num:
raise RuntimeError(f"与页面的连接已断开,没有获取到 ASIN 商品,重试{retry_num}/{max_retry_num}")
continue
# for sku_ele in sku_ls[0:2]:
for sku_ele in sku_ls:
# solve_problem = sku_ele.eles('xpath:.//kat-link[@label="解决商品信息问题"]')
asin = sku_ele.ele('xpath:.//div[contains(@class,"JanusSplitBox-module__container")]//div[contains(@class,"JanusSplitBox-module__panel--") and contains(string(.),"ASIN")]/..//div[last()]').text
self.log(f"*************** {asin} *****************")
self.log(f"ASIN {asin} 存在问题,正在点击解决...")
try:
if asin in self.already_asin:
self.log(f"{asin} 已经处理过了,跳过")
continue
xp_ls = [f'.//kat-link[@label="{i}"]' for i in ["解决商品信息问题","修复被禁止显示的商品","解决商品移除风险","查看冻结原因"]]
xp = "|".join(xp_ls)
# solve_problem = sku_ele.eles('xpath:.//kat-link[@label="解决商品信息问题"]|.//kat-link[@label="修复被禁止显示的商品"]|.//kat-link[@label="解决商品移除风险"]')
solve_problem = sku_ele.eles(f'xpath:{xp}')
if len(solve_problem) == 0:
# //a[text0=”添加缺失的商品详情”]
lack_info = sku_ele.eles('xpath:.//a[text()="添加缺失的商品详情"]')
if len(lack_info) > 0:
self.log(f"{asin} 需要添加缺失的商品详情,跳过...")
yield (asin,"添加缺失的商品详情")
continue
self.log(f"{asin},没有处理入口,不处理")
yield (asin,"没有处理入口,不处理")
self.already_asin.add(asin)
continue
solve_problem[0].click()
label = solve_problem[0].attr('label')
self.log(f"【当前类别】{label}")
action_panel = self.tab.ele('xpath://kat-panel-wrapper[@data-testid="kat-panel-wrapper-ActionPanelContent"]')
action_panel.wait.displayed(timeout=5, raise_err=False)
self.wait_loaded()
time.sleep(1)
# 如果存在“请求批准”按钮,则直接返回跳过
request_approval_btn = self.tab.eles('xpath://kat-panel-wrapper[@data-testid="kat-panel-wrapper-ActionPanelContent"]//div[@data-testid="section-header" and contains(string(.),"移除")]',timeout=5)
if len(request_approval_btn) > 0:
# 新增"查看冻结原因" 的情况下,需要点击修复商品详情
if "查看冻结原因" in label:
repair_produ_btn_xp = '//kat-panel-wrapper[@data-testid="kat-panel-wrapper-ActionPanelContent"]//kat-button[@variant="secondary" and @class="action-button" and contains(string(.),"修复商品信息")]'
repair_produ_btn = self.tab.eles(f'xpath:{repair_produ_btn_xp}',timeout=5)
self.log(f"{asin}【查看冻结原因】修复商品信息数量,{len(repair_produ_btn)}")
if len(repair_produ_btn) > 0:
self.log(f"{asin}【查看冻结原因】修复商品信息数量,开始操作")
self.clear_tab()
repair_produ_btn[0].click()
time.sleep(3)
# 获取最新的tab
new_tab = self.browser.latest_tab
if isinstance(new_tab, str):
new_tab = self.browser.get_tab(id_or_num=new_tab)
self.handle_repair_product(new_tab,headle_type="查看冻结原因")
else:
self.log(f"{i}【查看冻结原因】修复商品信息没有,不需要操作")
else:
self.log(f"ASIN {asin} 存在请求批准按钮,不需要处理,跳过...")
panel_close_btn = self.tab.ele('xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]',
timeout=5).sr('xpath:.//button[@class="close"]')
panel_close_btn.click()
action_panel.wait.deleted(timeout=3, raise_err=False)
yield (asin,"请求批准")
self.already_asin.add(asin)
continue
# 无需采取任何操作
not_operate = self.tab.eles('xpath://kat-alert[contains(@description,"如果您之前已提交更改,则这些更改当前正在处理中") and not(@dismissed)]',timeout=3)
if len(not_operate) > 0:
self.log(f"ASIN {asin} 无需采取任何操作,跳过...")
yield (asin,"无需操作")
self.already_asin.add(asin)
continue
# 解决商品信息违规问题 按钮
kat_box_ls = action_panel.eles('xpath:.//kat-box[@variant="white"]',timeout=5)
# if len(kat_box_ls) == 0:
# print(f"ASIN {asin} 的操作面板中未找到选项,跳过...")
# continue
self.log(f"ASIN {asin}需要处理的有:{len(kat_box_ls)}个问题.")
# 解决商品信息违规问题 按钮,有多少个都要处理
for i in range(len(kat_box_ls)):
self.log(f"开始处理第{i}个问题")
if len(kat_box_ls) <= i:
self.log(f"ASIN {asin} 的操作面板中选项数量发生变化,停止处理...")
break
kat_box_ls[i].click()
time.sleep(0.5)
self.wait_loaded()
# TODO 增加 “出现警告和安全信息时候下滑” 的情况
safe_handle_ls = self.tab.eles('xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]//div[@aria-label="安全证明"]//kat-label[@text="未开始"]',
timeout=15)
if len(safe_handle_ls) > 0:
self.log("存在安全证明,正在处理...")
safe_handle_ls[0].click()
time.sleep(0.5)
self.wait_loaded()
check_box = self.tab.ele('xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]//kat-checkbox')
check_box.click()
else:
problem_selection_ls = self.tab.eles(
'xpath://kat-panel-wrapper[@data-testid="kat-panel-wrapper-ActionPanelContent"]//div[@data-testid="registry-list"]//div[@data-testid="registry"]',
timeout=10)
self.log(f"待选择数量,{len(problem_selection_ls)}")
for problem_selection in problem_selection_ls:
problem_selection.click()
print("点击选择完成")
time.sleep(0.5)
# 点击保存
confirm_btn = self.tab.eles('xpath://kat-panel-wrapper[@data-testid="kat-panel-wrapper-ActionPanelContent"]//kat-button[@variant="primary"]')
if len(confirm_btn) > 0:
confirm_btn[0].wait.displayed(timeout=3, raise_err=False)
confirm_btn[0].click()
self.wait_loaded()
# 等待关闭按钮出现
try:
close_btn = self.tab.eles('xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]//kat-button[@label="关闭"]',
timeout=10)
if len(close_btn)>0:
close_btn[0].click()
except Exception as e:
self.log(f"点击关闭按钮失败,{e}")
kat_box_ls = action_panel.eles('xpath:.//kat-box[@variant="white"]', timeout=5)
# 修复商品信息
repair_product = action_panel.eles('xpath:.//kat-button[@class="action-button" and not(@variant="secondary")]',timeout=5)
if len(repair_product) > 0:
self.log(f"ASIN {asin} 存在修复商品信息按钮,正在点击...")
self.clear_tab()
repair_product[0].click()
time.sleep(3)
# 获取最新的tab
new_tab = self.browser.latest_tab
if isinstance(new_tab, str):
new_tab = self.browser.get_tab(id_or_num=new_tab)
self.handle_repair_product(new_tab)
yield (asin,"处理完成")
self.already_asin.add(asin)
self.log(f"*********************{asin} 处理完成*************************")
try:
# 全部操作完成,关闭
panel_close_btn= self.tab.ele('xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]',
timeout=5).sr('xpath:.//button[@class="close"]')
panel_close_btn.click()
action_panel.wait.deleted(timeout=3, raise_err=False)
self.clear_tab()
except Exception as e:
self.log(f"关闭操作面板失败,{e}")
except Exception as e:
yield (asin, "处理失败")
self.already_asin.add(asin)
self.log(f"{asin} 操作出错:{traceback.format_exc()}")
# 判断是否存在需要翻页的情况
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
if len(page_pamel) == 0:
break
next_page_btn = page_pamel[0].sr('xpath:.//span[@part="pagination-nav-right"]')
class_str = next_page_btn.attr('class')
if "end" in class_str:
break
next_page_btn.click()
num += 1
self.log(f"【程序计算】正在翻页,已翻 {num} 页...")
# already_asin = set()
except Exception as e:
self.log(f"处理审批操作异常,{e}")
traceback.print_exc()
retry_num += 1
self.tab.refresh()
self.tab.wait.doc_loaded(raise_err=False,timeout=120)
class ApproveTask(TaskBase):
"""审批任务处理类:负责处理产品风险审批任务"""
task_name = "产品风险审批-TASK"
def process_task(self, task_data: dict):
"""处理审批任务主入口
Args:
task_data: 任务数据
"""
try:
data = task_data.get("data", {})
task_id = data.get("taskId")
items = data.get("items", [])
country_codes = data.get("country_codes", [])
risk_listing_filter = data.get("risk_listing_filter", "")
# 测试
# risk_listing_filter = "DetailPageRemoved" #data.get("risk_listing_filter", "")
if not task_id:
self.log("任务ID为空跳过", "WARNING")
return
if not items:
self.log("店铺列表为空,跳过", "WARNING")
return
if not country_codes:
self.log("国家列表为空,跳过", "WARNING")
return
self.log(f"开始处理审批任务 {task_id},共 {len(items)} 个店铺,{len(country_codes)} 个国家")
runing_task[task_id] = {
"status": "running",
"start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"total_shops": len(items),
"processed_shops": 0,
"total_countries": len(country_codes) * len(items),
"processed_countries": 0,
"total_asins": 0,
"processed_asins": 0,
"success_count": 0,
"failed_count": 0,
"stop_requested": False
}
# 遍历处理每个店铺
for idx, shop_item in enumerate(items, 1):
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理", "WARNING")
runing_task[task_id]["status"] = "stopped"
return
shop_name = shop_item.get("shopName", "未知店铺")
self.log(f"[{idx}/{len(items)}] 开始处理店铺: {shop_name}")
show_notification(f"开始处理店铺: {shop_name}", "info")
try:
self.process_shop(shop_item, country_codes, task_id,risk_listing_filter)
# 更新已处理店铺数
if task_id in runing_task:
runing_task[task_id]["processed_shops"] += 1
except Exception as e:
self.log(f"处理店铺 {shop_name} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 更新任务状态
if task_id in runing_task:
if runing_task[task_id].get("stop_requested", False):
runing_task[task_id]["status"] = "stopped"
self.log(f"任务 {task_id} 已被暂停!")
else:
runing_task[task_id]["status"] = "completed"
self.log(f"任务 {task_id} 处理完成!")
except Exception as e:
self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR")
if task_id:
if task_id in runing_task:
runing_task[task_id]["status"] = "failed"
runing_task[task_id]["error"] = str(e)
def process_shop(self, shop_item: dict, country_codes: list, task_id: int, risk_listing_filter: str):
"""处理单个店铺
Args:
shop_item: 店铺信息
country_codes: 国家代码列表
task_id: 任务ID
risk_listing_filter: 风险商品筛选条件
"""
shop_name = shop_item.get("shopName", "未知店铺")
company_name = shop_item.get("companyName", "")
if not company_name:
self.log(f"店铺 {shop_name} 的公司名称为空,跳过", "WARNING")
return
if task_id in runing_task:
runing_task[task_id]["current_shop"] = shop_name
# 将店铺添加到正在执行中的店铺列表
start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
runing_shop[shop_name] = start_time
self.log(f"店铺 {shop_name} 已添加到执行列表,账号: {company_name},开始时间: {start_time}")
# 店铺打开重试最多3次
driver = None
max_retries = 3
iskill = False
# 处理每个国家
for country_code in country_codes:
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理国家", "ERROR")
driver.close_store()
break
# 打开店铺
driver = self.open_shop(cls=AmzoneApprove,max_retries=max_retries, company_name=company_name,
shop_name=shop_name, iskill=iskill)
if driver is None:
self.log(f"任务 {task_id} 启动店铺失败,结束任务", "ERROR")
for country_code in country_codes:
self.post_result(task_id, shop_name, country_code, "", "", is_done=True)
return
current_url = None
for _ in range(max_retries):
try:
self.process_country(driver, country_code, task_id, shop_name,risk_listing_filter,target_url=current_url)
driver.reset_already_asin()
driver.close_store()
break
except Exception as e:
self.log(f"处理国家 {country_code} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
if "与页面的连接已断开" in str(e):
iskill = True
current_url = driver.url
# 更新已处理国家数
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
self.log(f"{task_id}任务处理完成")
# 最后回传,标记完成
try:
self.post_result(task_id, shop_name, country_code, "", "", is_done=True)
driver.close_store() #完成退出店铺
except Exception as e:
self.log(f"回传结果失败: {str(e)}", "ERROR")
# 从正在执行中的店铺列表中移除
if shop_name in runing_shop:
del runing_shop[shop_name]
self.log(f"店铺 {shop_name} 已从执行列表中移除")
def process_country(self, driver: AmzoneApprove, country_code: str, task_id: int,
shop_name: str, risk_listing_filter: str,target_url:str=None):
"""
处理单个国家的审批任务
"""
# 转换国家代码为中文名称
country_name = self.country_info.get(country_code, country_code)
info_mes = f"开始处理国家: {country_name} ({country_code})"
self.log(info_mes)
show_notification(info_mes, "info")
# 更新当前处理的国家
if task_id in runing_task:
runing_task[task_id]["current_country"] = country_name
# 切换国家最多重试3次
max_retries = 5
switch_success, switch_success_pg, search_success,sku_ls = self.action_init(driver, country_name, shop_name,
risk_listing_filter)
# 如果切换失败,直接返回
if not switch_success or not switch_success_pg or not search_success:
error_message = f"切换到国家({switch_success})/库存页面({switch_success_pg})/搜索筛选到指定选项({search_success}) {country_name} 失败,已重试 {max_retries} 次,跳过该国家"
self.log(error_message, "ERROR")
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
return
# # 如果没有需要审批的商品,直接返回
if len(sku_ls) == 0:
self.log(f"国家 {country_name} 没有需要审批的商品")
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
return
self.log(f"国家 {country_name}{len(sku_ls)} 个需要审批的商品,开始处理...")
if target_url is not None: #从失败的链接继续
self.log(f"开始访问:{target_url}")
driver.tab.get(url=target_url)
driver.tab.wait.doc_loaded(timeout=60,raise_err=False)
result = []
# 处理所有需要审批的商品通过yield获取结果
for asin, status in driver.run_page_action():
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求停止处理ASIN", "WARNING")
break
self.log(f"ASIN {asin} 处理结果: {status}")
if status in ["请求批准", "添加缺失的商品详情"]:
country_data = {
"status": "",
"shopName": shop_name,
"productAsinSku": "",
"done": False,
"removeAsin": asin,
"removeStatus": status
}
else:
country_data = {
"status": status,
"shopName": shop_name,
"productAsinSku": asin,
"done": False,
"removeAsin": "",
"removeStatus": ""
}
result.append(country_data)
if len(result) > 20:
self.post_result_batch(task_id, shop_name, country_code, result)
result = []
if len(result) > 0:
self.post_result_batch(task_id, shop_name, country_code, result)
self.log(f"国家 {country_name} 处理完成")
def post_result(self, task_id: int, shop_name: str, country_code: str, asin: str, status: str,is_done: bool = False):
"""回传处理结果到API
Args:
task_id: 任务ID
shop_name: 店铺名称
country_code: 国家代码
asin: ASIN
status: 处理状态
"""
url = f"{DELETE_BRAND_API_BASE}/api/product-risk-resolve/tasks/{task_id}/result"
if status in ["请求批准","添加缺失的商品详情"]:
country_data = {
"status": "",
"shopName": shop_name,
"productAsinSku": "",
"done": is_done,
"removeAsin": asin,
"removeStatus": status
}
else:
country_data = {
"status": status,
"shopName": shop_name,
"productAsinSku": asin,
"done": is_done,
"removeAsin": "",
"removeStatus": ""
}
payload = {
"shops": [
{
"error": "",
"countries": {
country_code : [ country_data ]
},
"shopName": shop_name
}
]
}
max_retries = 3
for retry in range(max_retries):
try:
self.log("=====================================")
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
self.log(f"回传URL: {url}")
self.log(f"回传数据: {payload}")
response = requests.post(
url,
json=payload,
headers={"Content-Type": "application/json"},
timeout=30,
verify=False
)
self.log(f"回传结果: {response.text}")
if response.status_code == 200:
self.log(f"结果回传成功: {asin} - {status}")
return
else:
self.log(f"结果回传失败,状态码: {response.status_code}", "WARNING")
print("=====================================")
except Exception as e:
self.log(f"调用API异常: {str(e)}", "ERROR")
print("=====================================")
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(2)
self.log(f"已达到最大重试次数,结果回传最终失败", "ERROR")
def post_result_batch(self, task_id: int, shop_name: str, country_code: str, country_data_ls:list):
"""
批量回传处理结果到API
"""
url = f"{DELETE_BRAND_API_BASE}/api/product-risk-resolve/tasks/{task_id}/result"
payload = {
"shops": [
{
"error": "",
"countries": {
country_code: country_data_ls
},
"shopName": shop_name
}
]
}
max_retries = 3
for retry in range(max_retries):
try:
self.log("=====================================")
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
self.log(f"回传URL: {url}")
self.log(f"回传数据: {payload}")
response = requests.post(
url,
json=payload,
headers={"Content-Type": "application/json"},
timeout=30,
verify=False
)
self.log(f"回传结果: {response.text}")
if response.status_code == 200:
self.log(f"结果回传成功")
return
else:
self.log(f"结果回传失败,状态码: {response.status_code}", "WARNING")
print("=====================================")
except Exception as e:
self.log(f"调用API异常: {str(e)}", "ERROR")
print("=====================================")
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(2)
self.log(f"已达到最大重试次数,结果回传最终失败", "ERROR")
if __name__ == '__main__':
user_info = {
"company": "rongchuang123",
"username": "自动化_Robot",
"password": "#20zsg25"
}
shop_name = "郭亚芳"
country = "德国"
kill_process('v6')
driver = AmzoneApprove(user_info)
browser = driver.open_shop(shop_name)
sw_suc = driver.SwitchingCountries(country)
driver.SwitchPage()
risk_listing_filter = "DetailPageRemoved"
for _ in range(3):
try:
sku_ls = driver.search(filter_type=risk_listing_filter)
break
except Exception as e:
print(e)
driver.tab.refresh()
if len(sku_ls) > 0:
print("有数据,开始操作")
for asin, status in driver.run_page_action():
print(f"ASIN {asin} 的处理结果: {status}")
print("已完成操作")
# for i in range(0):
# print(i)

View File

@@ -1,562 +0,0 @@
import time
import traceback
from datetime import datetime
import requests
from amazon.amazon_base import AmamzonBase, kill_process,TaskBase
from amazon.tool import show_notification
from config import runing_task, runing_shop, DELETE_BRAND_API_BASE
class AmzonePStatus(AmamzonBase):
mark_name = "状态查询"
def search_asin(self, asin):
search_region = self.tab.ele('xpath://div[@id="searchBoxContainer"]//kat-input-group')
search_region.wait.displayed(raise_err=False)
time.sleep(0.6)
search_input = self.tab.ele("xpath://kat-input[contains(@class,'SearchBox-module__searchInput')]").sr(
'xpath://span[@class="container"]//input[@part="input"]')
search_input.input(asin, clear=True)
sku_ls = []
for _ in range(3):
search_btn = self.tab.ele("xpath://kat-icon[@name='search']")
search_btn.click()
load_ele = self.tab.ele("xpath://div[contains(@class,'Loader-module__loader')]")
# load_ele.wait.hidden(timeout=3, raise_err=False)
load_ele.wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
sku_ls = self.tab.eles("xpath://div[@data-sku]", timeout=3)
if len(sku_ls) > 0:
break
return sku_ls
def search_asin_action(self, asin: str):
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]", timeout=5)
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
sku_ls = self.search_asin(asin=asin)
print(f"{self.mark_name}{asin} 搜索到 {len(sku_ls)} 个SKU")
def clear_tab(self):
all_tab = self.browser.get_tabs()
close_tab = []
for tab in all_tab:
tab_id = tab if isinstance(tab, str) else tab.tab_id
if self.tab.tab_id == tab_id:
continue
close_tab.append(tab)
print("需要关闭的标签页", close_tab)
print("当前操作的tab_id", self.tab.tab_id)
self.browser.close_tabs(close_tab)
def run_page_action(self, appoint_asin: str = None):
self.log(f"==================={self.mark_name}=======================")
self.log("开始执行...")
num = 0
retry_num = 0
already_asin = set()
get_page_faild = 0
while retry_num < 3: # 最多重试3次
try:
print(self.mark_name, "指定asin操作", appoint_asin)
self.search_asin_action(asin=appoint_asin)
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]", timeout=5)
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
# 获取当前页码
try:
page_pamel = self.tab.eles('xpath://kat-pagination', timeout=5)
if len(page_pamel) > 0:
current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text
# 总页数
total_page = page_pamel[0].sr.eles(
'xpath:.//ul[@class="pages"]//span[@class="page__inner"][last()]')
if len(total_page) > 0:
total_page = total_page[-1].text
else:
total_page = 0
print(f"{self.mark_name}】当前页码: {current_page} / 总页数: {total_page}")
get_page_faild = 0
except Exception as e:
print("{self.mark_name}", "获取页码失败", e)
get_page_faild += 1
if get_page_faild > 2:
show_notification(f"{self.mark_name}】获取页码失败超3次停止任务")
break
sku_ls = self.tab.eles("xpath://div[@data-sku]", timeout=10)
print(f"{self.mark_name}】获取到 {len(sku_ls)}")
# for sku_ele in sku_ls[0:2]:
if len(sku_ls) == 0:
yield (appoint_asin,"查询不到")
break
for sku_ele in sku_ls:
asin = sku_ele.ele(
'xpath:.//div[contains(@class,"JanusSplitBox-module__container")]//div[contains(@class,"JanusSplitBox-module__panel--") and contains(string(.),"ASIN")]/..//div[last()]',
timeout=3).text
print(f"{self.mark_name}】ASIN {asin} 找到....")
if asin in already_asin:
print(f"{self.mark_name}{asin} 已经处理过了,跳过")
continue
# 状态
status_ele = sku_ele.eles('xpath:.//div[contains(@class,"Status-module__container")]//kat-label')
if len(status_ele) > 0:
statu = status_ele[0].attr("emphasis")
else:
statu = "未获取到状态"
already_asin.add(asin)
yield (asin,statu)
print(f"{self.mark_name}",(asin,statu))
# 判断是否存在需要翻页的情况
page_pamel = self.tab.eles('xpath://kat-pagination', timeout=5)
if len(page_pamel) == 0:
break
next_page_btn = page_pamel[0].sr('xpath:.//span[@part="pagination-nav-right"]')
class_str = next_page_btn.attr('class')
if "end" in class_str:
break
next_page_btn.click()
num += 1
print(f"{self.mark_name}】【程序计算】正在翻页,已翻 {num} 页...")
already_asin = set()
except Exception as e:
print(f"{self.mark_name}】处理跟价操作异常", e)
traceback.print_exc()
retry_num += 1
self.tab.refresh()
self.tab.wait.doc_loaded(raise_err=False, timeout=120)
class StatusTask(TaskBase):
task_name = "状态查询-TASK"
def process_task(self, task_data: dict):
"""处理审批任务主入口
Args:
task_data: 任务数据
"""
try:
data = task_data.get("data", {})
task_id = data.get("taskId")
items = data.get("items", [])[0]
shop_name = items.get("shop_name")
queryAsins = items.get("queryAsins", [])
user_id = data.get("user_id")
stage_index = data.get("stage_index")
final_stage = bool(data.get("final_stage", True))
# 用于测试
limit = data.get("limit", None)
if not task_id or not queryAsins:
self.log("任务ID / queryAsins列表 为空,跳过", "WARNING")
return
self.log(f"开始处理审批任务 {task_id},共 1 个店铺,{len(queryAsins)} 个国家")
runing_task[task_id] = {
"status": "running",
"start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"total_shops": 1,
"processed_shops": 0,
"total_countries": len(queryAsins),
"processed_countries": 0,
"total_asins": 0,
"processed_asins": 0,
"success_count": 0,
"failed_count": 0,
"stop_requested": False
}
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理", "WARNING")
runing_task[task_id]["status"] = "stopped"
return
self.log(f"开始处理店铺: {shop_name}")
show_notification(f"开始处理店铺: {shop_name}", "info")
try:
self.process_shop(items, queryAsins, task_id, user_id, stage_index,
final_stage, limit=limit)
# self.process_shop(shop_item, country_codes, task_id,risk_listing_filter)
# 更新已处理店铺数
if task_id in runing_task:
runing_task[task_id]["processed_shops"] += 1
except Exception as e:
self.log(f"处理店铺 {shop_name} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 更新任务状态
if task_id in runing_task:
if runing_task[task_id].get("stop_requested", False):
runing_task[task_id]["status"] = "stopped"
self.log(f"任务 {task_id} 已被暂停!")
else:
runing_task[task_id]["status"] = "completed"
self.log(f"任务 {task_id} 处理完成!")
except Exception as e:
self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR")
if task_id:
if task_id in runing_task:
runing_task[task_id]["status"] = "failed"
runing_task[task_id]["error"] = str(e)
def process_shop(self, shop_item: dict, country_codes: list, task_id: int,
user_id=None, stage_index=None, final_stage: bool = True, limit: str = None):
"""处理单个店铺
Args:
shop_item: 店铺信息
country_codes: 国家代码列表
task_id: 任务ID
risk_listing_filter: 风险商品筛选条件
"""
shop_name = shop_item.get("shopName", "未知店铺")
company_name = shop_item.get("companyName", "")
if not company_name:
self.log(f"店铺 {shop_name} 的公司名称为空,跳过", "WARNING")
return
if task_id in runing_task:
runing_task[task_id]["current_shop"] = shop_name
# 将店铺添加到正在执行中的店铺列表
start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
runing_shop[shop_name] = start_time
self.log(f"店铺 {shop_name} 已添加到执行列表,账号: {company_name},开始时间: {start_time}")
# 店铺打开重试最多3次
max_retries = 3
iskill = False
# 处理每个国家
for value in country_codes:
country_code = value.get("country")
all_asin = value.get("asins")
# 打开店铺
driver = self.open_shop(cls=AmzonePStatus,max_retries=max_retries, company_name=company_name,
shop_name=shop_name, iskill=iskill)
if driver is None:
self.log(f"任务 {task_id} 启动店铺失败,结束任务", "ERROR")
for country_code in country_codes:
self.post_result(task_id, shop_name, country_code, "", "", is_done=True)
return
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理国家", "ERROR")
driver.close_store()
break
try:
self.process_country(driver, country_code, task_id, shop_name,all_asin=all_asin )
except Exception as e:
self.log(f"处理国家 {country_code} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
if "与页面的连接已断开" in str(e):
iskill = True
# 更新已处理国家数
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
# 关闭店铺
driver.close_store()
self.log(f"{task_id}任务处理完成")
# 最后回传,标记完成
try:
# self.post_result(task_id, shop_name, country_code, "", "", is_done=True)
if final_stage:
self.post_result(task_id, shop_name, country_code, "", "", is_done=True)
else:
self.post_stage_finished(task_id, user_id, stage_index)
except Exception as e:
self.log(f"回传结果失败: {str(e)}", "ERROR")
# 从正在执行中的店铺列表中移除
if shop_name in runing_shop:
del runing_shop[shop_name]
self.log(f"店铺 {shop_name} 已从执行列表中移除")
def process_country(self, driver: AmzonePStatus, country_code: str, task_id: int, shop_name: str,
all_asin:list,limit: str = None):
"""处理单个国家的审批任务
Args:
driver: AmzoneApprove驱动实例
country_code: 国家代码(如 UK, DE, FR 等)
task_id: 任务ID
shop_name: 店铺名称
risk_listing_filter: 风险商品筛选条件
"""
# 转换国家代码为中文名称
country_name = self.country_info.get(country_code, country_code)
info_mes = f"开始处理国家: {country_name} ({country_code})"
self.log(info_mes)
show_notification(info_mes, "info")
# 更新当前处理的国家
if task_id in runing_task:
runing_task[task_id]["current_country"] = country_name
# 切换国家最多重试3次
max_retries = 3
switch_success = False
for retry in range(max_retries):
try:
self.log(f"尝试切换到国家 {country_name} (第 {retry + 1}/{max_retries} 次)")
if retry > 1:
# 刷新不行就重新打开店铺
self.log("重试前重新打开店铺...")
try:
driver.close_store()
time.sleep(3)
driver.open_shop(shop_name)
except Exception as e:
self.log(f"关闭重新打开店铺: {str(e)}", "WARNING")
# 如果不是第一次尝试,先刷新页面
if retry > 0:
self.log("重试前刷新页面...")
try:
driver.tab.refresh()
time.sleep(3)
except Exception as e:
self.log(f"刷新页面失败: {str(e)}", "WARNING")
switch_success = driver.SwitchingCountries(country_name)
if switch_success:
self.log(f"成功切换到国家 {country_name}")
break
else:
self.log(f"切换到国家 {country_name} 失败", "WARNING")
except Exception as e:
self.log(f"切换国家 {country_name} 异常: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(2)
# 如果切换失败,直接返回
if not switch_success:
error_message = f"切换到国家 {country_name} 失败,已重试 {max_retries} 次,跳过该国家"
self.log(error_message, "ERROR")
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
return
# 切换到库存管理页面
for retry in range(max_retries):
try:
driver.SwitchPage()
self.log(f"已切换到库存管理页面")
except Exception as e:
self.log(f"切换页面失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
if retry >= max_retries - 1:
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
self.log(f"切换页面失败重试退出", "ERROR")
return
# 处理所有需要审批的商品通过yield获取结果
# 指定 asin
for ap_asin in all_asin:
print(f"{driver.mark_name}】开始处理 ->",ap_asin)
for asin, status in driver.run_page_action(
appoint_asin=ap_asin
):
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求停止处理ASIN", "WARNING")
break
self.log(f"ASIN {asin} 处理结果: {status}")
# 更新任务状态
if task_id in runing_task:
runing_task[task_id]["current_asin"] = asin
runing_task[task_id]["processed_asins"] += 1
runing_task[task_id]["failed_count"] += 1
# 回传结果到API
try:
self.post_result(task_id, shop_name, country_code, asin, status)
except Exception as e:
self.log(f"回传结果失败: {str(e)}", "ERROR")
self.log(f"国家 {country_name} 处理完成")
def post_stage_finished(self, task_id: int, user_id, stage_index):
# import requests
# from config import DELETE_BRAND_API_BASE
if user_id in (None, "", 0):
raise ValueError("user_id is required for stage completion callback")
if stage_index is None:
raise ValueError("stage_index is required for stage completion callback")
url = f"{DELETE_BRAND_API_BASE}/api/shop-match/tasks/{task_id}/stage-finished"
payload = {"stage_index": stage_index}
params = {"user_id": user_id}
max_retries = 3
for retry in range(max_retries):
try:
self.log(f"Attempting stage completion callback ({retry + 1}/{max_retries})")
response = requests.post(
url,
params=params,
json=payload,
headers={"Content-Type": "application/json"},
timeout=30,
verify=False,
)
self.log(f"Stage completion callback response: {response.text}")
data = response.json() if response.text else {}
if response.status_code == 200 and isinstance(data, dict) and data.get("success"):
self.log(f"Stage completion callback succeeded: task={task_id}, stage={stage_index}")
return
self.log(f"Stage completion callback failed, status={response.status_code}", "WARNING")
except Exception as e:
self.log(f"Stage completion callback exception: {str(e)}", "ERROR")
if retry < max_retries - 1:
time.sleep(2)
raise RuntimeError(f"Stage completion callback failed after retries: task={task_id}, stage={stage_index}")
def post_result(self, task_id: int, shop_name: str, country_code: str, asin: str, status: dict,
is_done: bool = False):
"""回传处理结果到API
Args:
task_id: 任务ID
shop_name: 店铺名称
country_code: 国家代码
asin: ASIN
status: 处理状态
"""
url = f"{DELETE_BRAND_API_BASE}/api/query-asin/tasks/{task_id}/result"
payload = {
"shops": [
{
"shopName": shop_name,
"countryResults": [
{
"country": country_code,
"items": [
{
"asin": asin,
"status": status
}
]
}
],
"shopDone": is_done,
"submissionId": f"{int(time.time())}"
}
]
}
if is_done:
payload["shops"][0]["success"] = is_done
max_retries = 3
for retry in range(max_retries):
try:
print("================【跟价】=====================")
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
self.log(f"回传URL: {url}")
self.log(f"回传数据: {payload}")
response = requests.post(
url,
json=payload,
headers={"Content-Type": "application/json"},
timeout=30,
verify=False
)
self.log(f"回传结果: {response.text}")
data = response.json() if response.text else {}
if response.status_code == 200 and isinstance(data, dict) and data.get("success"):
self.log(f"结果回传成功: {asin} - {status}")
return
else:
self.log(f"结果回传失败,状态码: {response.status_code}", "WARNING")
print("=====================================")
except Exception as e:
self.log(f"调用API异常: {str(e)}", "ERROR")
print("=====================================")
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(2)
self.log(f"已达到最大重试次数,结果回传最终失败", "ERROR")
raise RuntimeError("已达到最大重试次数,结果回传最终失败")
if __name__ == '__main__':
# 使用示例
user_info = {
"company": "rongchuang123",
"username": "自动化_Robot",
"password": "#20zsg25"
}
shop_name = "魏振峰"
country = "德国"
kill_process('v6')
driver = AmzonePStatus(user_info)
browser = driver.open_shop(shop_name)
sw_suc = driver.SwitchingCountries(country)
driver.SwitchPage()
risk_listing_filter = "Active"
_shopMallName = "WEIZHENFENG168"
ap_asin = "B0F1N18XFW"
skip_asin = []
for _ in range(3):
try:
sku_ls = driver.search(filter_type=risk_listing_filter)
break
except Exception as e:
print(e)
driver.tab.refresh()
if len(sku_ls) > 0:
print("有数据,开始操作")
for asin, status in driver.run_page_action(
current_shop_name=_shopMallName,
appoint_asin=ap_asin, skip_asin=skip_asin
):
print(f"ASIN {asin} 的处理结果: {status}")
print("已完成操作")

View File

@@ -1,808 +0,0 @@
import json
import os
import re
import subprocess
import time
import uuid
from typing import Any, Dict, List, Literal, Optional, TypedDict
from loguru import logger
import requests
from DrissionPage import Chromium
from DrissionPage._pages.chromium_tab import ChromiumTab
from DrissionPage.common import By
try:
import winreg
except ImportError:
winreg = None
STATUS_OK = "0"
STATUS_LOGIN_FAILED = "-10003"
DEFAULT_SOCKET_PORT = 20000
CLIENT_API_TIMEOUT = 120
PORT_CHECK_TIMEOUT = 2
UPDATE_CORE_RETRY_DELAY = 2
CLIENT_RESTART_DELAY = 5
CLIENT_START_RETRIES = 3
CLIENT_READY_TIMEOUT = 10
CLIENT_READY_INTERVAL = 0.5
CLIENT_POST_START_DELAY = 5
PROCESS_KILL_DELAY = 3
ZINIAO_WEBDRIVER_LOG_TAIL_LINES = 80
DOC_LOAD_TIMEOUT = 30
COUNTRY_INITIAL_LOAD_TIMEOUT = 120
COUNTRY_LOOKUP_TIMEOUT = 20
COUNTRY_CLICK_TIMEOUT = 10
COUNTRY_DROPDOWN_DELAY = 1
COUNTRY_DROPDOWN_READY_TIMEOUT = 20
COUNTRY_DROPDOWN_RETRY_INTERVAL = 0.5
IP_CHECK_TIMEOUT = 60
LOGIN_ATTEMPTS = 4
PASSWORD_INPUT_TIMEOUT = 5
PASSWORD_SUBMIT_TIMEOUT = 5
OTP_SEND_TIMEOUT = 5
OTP_INPUT_LOOKUP_ATTEMPTS = 5
OTP_INPUT_TIMEOUT = 30
OTP_INPUT_DISPLAY_TIMEOUT = 10
OTP_CODE_WAIT_SECONDS = 30
OTP_SUBMIT_TIMEOUT = 10
OTP_RESULT_LOAD_TIMEOUT = 20
OTP_ERROR_TIMEOUT = 10
OTP_SEND_DELAY = 1
LOGIN_FINAL_SUBMIT_TIMEOUT = 10
NEED_LOGIN_DELAY = 3
NEED_LOGIN_TIMEOUT = 5
COUNTRY_LABEL_XPATH = 'xpath://div[@class="dropdown-account-switcher-header-label"]/span[last()]'
COUNTRY_DROPDOWN_XPATH = 'xpath://div[@class="dropdown-account-switcher-header-label"]'
COUNTRY_LIST_ITEM_XPATH = 'xpath://div[@class="dropdown-account-switcher-list-item"]'
NEED_LOGIN_XPATH = 'xpath://h1[@class="a-spacing-small"]|//span[contains(text(),"登录")]'
PASSWORD_INPUT_XPATH = 'xpath://input[@type="password"]'
PASSWORD_SUBMIT_XPATH = 'xpath://input[@id="signInSubmit"]'
OTP_SEND_XPATH = 'xpath://span[@id="auth-send-code" and contains(string(.),"发送一次性密码")]'
OTP_INPUT_XPATH = 'xpath://input[@name="otpCode"]'
OTP_SUBMIT_XPATH = 'xpath://input[@id="auth-signin-button"]'
OTP_ERROR_XPATH = 'xpath://div[@id="auth-error-message-box"]'
class UserInfo(TypedDict):
username: str
password: str
company: str
def kill_process(version: Literal["v5", "v6"]):
"""结束指定版本的紫鸟客户端进程."""
logger.info("准备杀紫鸟客户端进程,version={}", version)
driver = ZiniaoDriver({})
driver.kill_process(version)
class ZiniaoDriver:
"""封装紫鸟客户端启动,店铺浏览器生命周期和客户端 API 调用."""
def __init__(self, user_info: UserInfo, socket_port: int = DEFAULT_SOCKET_PORT):
"""初始化紫鸟浏览器驱动.
Args:
user_info: 紫鸟账号信息,包含 company, username, password.
socket_port: 客户端 HTTP 通信端口.
"""
self.user_info = user_info
self.socket_port = socket_port
self.client_path = None
self.browser = None
self.tab: Optional[ChromiumTab] = None
self.store_id = None
@property
def client_url(self) -> str:
return f"http://127.0.0.1:{self.socket_port}"
def _build_payload(self, action: str, extra: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
payload = {
"action": action,
"requestId": str(uuid.uuid4()),
}
if extra:
payload.update(extra)
payload.update(self.user_info)
return payload
def _post_client(self, payload: Dict[str, Any]) -> Dict[str, Any]:
response = requests.post(
self.client_url,
json.dumps(payload).encode("utf-8"),
timeout=CLIENT_API_TIMEOUT,
)
return response.json()
@staticmethod
def _status_code(result: Optional[Dict[str, Any]]) -> Optional[str]:
if result is None:
return None
return str(result.get("statusCode"))
@staticmethod
def _result_text(result: Dict[str, Any]) -> str:
return json.dumps(result, ensure_ascii=False)
@staticmethod
def _find_store_oauth(shop_list: Optional[List[Dict[str, Any]]], shop_name: str):
for shop in shop_list or []:
if shop.get("browserName") == shop_name:
return shop.get("browserOauth")
return None
def _is_client_ready(self) -> bool:
try:
requests.get(self.client_url, timeout=PORT_CHECK_TIMEOUT)
return True
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
return False
def _wait_until_client_ready(self, timeout: float) -> bool:
start_check_time = time.time()
while time.time() - start_check_time < timeout:
if self._is_client_ready():
return True
time.sleep(CLIENT_READY_INTERVAL)
return False
@staticmethod
def _redact_ziniao_log_line(line: str) -> str:
return re.sub(r'("password"\s*:\s*)"[^"]*"', r'\1"***"', line)
@staticmethod
def _ziniao_webdriver_log_path() -> Optional[str]:
appdata = os.getenv("APPDATA")
if not appdata:
return None
filename = f"webdriver.{time.strftime('%Y%m%d')}.log"
return os.path.join(
appdata,
"ziniaobrowser",
"instances",
"userdata1",
"logs",
"client",
filename,
)
def _log_ziniao_webdriver_tail(self) -> None:
log_path = self._ziniao_webdriver_log_path()
if not log_path:
logger.warning("无法读取紫鸟 webdriver 日志: APPDATA 环境变量为空")
return
if not os.path.exists(log_path):
logger.warning("紫鸟 webdriver 日志不存在:{}", log_path)
return
try:
with open(log_path, "r", encoding="utf-8", errors="replace") as log_file:
lines = log_file.readlines()
except OSError as exc:
logger.warning("读取紫鸟 webdriver 日志失败:{} | {}", log_path, exc)
return
tail_lines = lines[-ZINIAO_WEBDRIVER_LOG_TAIL_LINES:]
tail_text = "".join(self._redact_ziniao_log_line(line) for line in tail_lines).strip()
if tail_text:
logger.error("紫鸟 webdriver 日志尾部({}):\n{}", log_path, tail_text)
else:
logger.warning("紫鸟 webdriver 日志为空:{}", log_path)
def get_zinaio_exe(self, protocol_name: str = "superbrowser"):
"""从 Windows 注册表读取紫鸟客户端可执行文件路径.
Args:
protocol_name: 注册表协议名称.
Returns:
紫鸟浏览器可执行文件路径. 未找到时返回 None.
"""
if winreg is None:
logger.error("当前系统不支持 winreg,无法从注册表获取紫鸟客户端路径")
return None
key_path = rf"SOFTWARE\Classes\{protocol_name}\shell\open\command"
for root_key, root_name in (
(winreg.HKEY_CURRENT_USER, "HKEY_CURRENT_USER"),
(winreg.HKEY_LOCAL_MACHINE, "HKEY_LOCAL_MACHINE"),
):
try:
logger.info("正在从注册表读取紫鸟客户端路径:{}\\{}", root_name, key_path)
key = winreg.OpenKey(root_key, key_path)
command, _ = winreg.QueryValueEx(key, "")
winreg.CloseKey(key)
if isinstance(command, str):
sub = "ziniao.exe"
exe_path = command[0 : command.find(sub) + len(sub) + 1]
else:
exe_path = command[0]
logger.info("已获取紫鸟客户端路径:{}", exe_path)
return exe_path
except FileNotFoundError:
logger.warning("注册表中未找到紫鸟客户端路径:{}\\{}", root_name, key_path)
logger.error("未能获取紫鸟客户端路径,protocol_name={}", protocol_name)
return None
def update_core(self):
"""更新紫鸟浏览器内核.
打开店铺前调用,要求客户端版本 5.285.7 以上. 接口可能因 HTTP 超时
返回未完成状态,因此会循环调用直到成功或判定客户端不支持.
"""
payload = self._build_payload("updateCore")
logger.info("开始更新紫鸟浏览器内核")
while True:
result = self._post_client(payload)
logger.info("更新内核返回:{}", result)
if self._handle_update_core_result(result):
return
if result is None:
continue
logger.info("等待更新内核完成:{}", self._result_text(result))
time.sleep(UPDATE_CORE_RETRY_DELAY)
def _handle_update_core_result(self, result: Optional[Dict[str, Any]]) -> bool:
if result is None:
logger.info("等待客户端启动后继续更新内核")
time.sleep(UPDATE_CORE_RETRY_DELAY)
return False
status_code = self._status_code(result)
if status_code is None or status_code == STATUS_LOGIN_FAILED:
logger.error("当前紫鸟客户端版本不支持更新内核接口,请升级客户端")
return True
if status_code == STATUS_OK:
logger.info("紫鸟浏览器内核更新完成")
return True
return False
def kill_process(self, version: Literal["v5", "v6"]):
"""结束指定版本的紫鸟客户端进程.
Args:
version: 客户端主版本.
"""
if version == "v5":
process_name = "SuperBrowser.exe"
logger.info("准备结束紫鸟 v5 starter.exe")
self._kill_process_by_name("starter.exe")
else:
process_name = "ziniao.exe"
logger.info("准备结束紫鸟客户端进程:{}", process_name)
self._kill_process_by_name(process_name)
time.sleep(PROCESS_KILL_DELAY)
def _kill_process_by_name(self, process_name: str) -> None:
result = subprocess.run(
["taskkill", "/f", "/t", "/im", process_name],
capture_output=True,
text=True,
encoding="utf-8",
errors="ignore",
check=False,
)
output = (result.stdout or result.stderr or "").strip()
if result.returncode == 0:
if output:
logger.info("结束进程返回:{}", output)
return
if "没有找到进程" in output or "not found" in output.lower():
logger.info("进程未运行,跳过结束:{}", process_name)
return
logger.warning("结束进程失败:{} | {}", process_name, output or f"returncode={result.returncode}")
def get_browser_list(self) -> Optional[list[Dict[str, Any]]]:
"""获取紫鸟浏览器店铺列表.
Returns:
浏览器店铺列表.登录失败或请求失败时返回 None.
"""
payload = self._build_payload("getBrowserList")
request_id = payload["requestId"]
logger.info("开始获取紫鸟浏览器列表,requestId={}", request_id)
result = self._post_client(payload)
status_code = self._status_code(result)
if status_code == STATUS_OK:
browser_list = result.get("browserList")
logger.info("获取紫鸟浏览器列表成功,数量={}", len(browser_list or []))
return browser_list
if status_code == STATUS_LOGIN_FAILED:
logger.error("获取紫鸟浏览器列表登录失败:{}", self._result_text(result))
return None
logger.error("获取紫鸟浏览器列表失败:{}", self._result_text(result))
return None
def open_store(
self,
store_info,
isWebDriverReadOnlyMode=0,
isprivacy=0,
isHeadless=0,
cookieTypeSave=0,
jsInfo="",
):
"""启动紫鸟店铺浏览器.
Args:
store_info: 店铺 browserId 或 browserOauth.
isWebDriverReadOnlyMode: WebDriver 只读模式开关.
isprivacy: 隐私模式开关.
isHeadless: 无头模式开关.
cookieTypeSave: Cookie 保存类型.
jsInfo: 注入的 JS 信息.
Returns:
紫鸟客户端返回的启动结果.
"""
payload = self._build_start_browser_payload(
store_info=store_info,
is_web_driver_read_only_mode=isWebDriverReadOnlyMode,
is_privacy=isprivacy,
is_headless=isHeadless,
cookie_type_save=cookieTypeSave,
js_info=jsInfo,
)
request_id = payload["requestId"]
logger.info("开始打开紫鸟店铺,store_info={},requestId={}", store_info, request_id)
result = self._post_client(payload)
status_code = self._status_code(result)
if status_code == STATUS_OK:
logger.info("紫鸟店铺打开成功,debuggingPort={}", result.get("debuggingPort"))
return result
if status_code == STATUS_LOGIN_FAILED:
logger.error("打开紫鸟店铺登录失败:{}", self._result_text(result))
raise RuntimeError(f"[open_store]登录失败 {self._result_text(result)}")
logger.error("打开紫鸟店铺失败:{}", self._result_text(result))
raise RuntimeError(f"[open_store]失败 {self._result_text(result)} ")
def _build_start_browser_payload(
self,
*,
store_info,
is_web_driver_read_only_mode=0,
is_privacy=0,
is_headless=0,
cookie_type_save=0,
js_info="",
) -> Dict[str, Any]:
payload = self._build_payload(
"startBrowser",
{
"isWaitPluginUpdate": 0,
"isHeadless": is_headless,
"isWebDriverReadOnlyMode": is_web_driver_read_only_mode,
"cookieTypeLoad": 0,
"cookieTypeSave": cookie_type_save,
"runMode": "1",
"isLoadUserPlugin": True,
"pluginIdType": 1,
"privacyMode": is_privacy,
},
)
if store_info.isdigit():
payload["browserId"] = store_info
else:
payload["browserOauth"] = store_info
if len(str(js_info)) > 2:
payload["injectJsInfo"] = json.dumps(js_info)
return payload
def get_browser(self, port) -> Chromium:
"""连接 DrissionPage 浏览器实例.
Args:
port: Chromium 调试端口.
Returns:
DrissionPage Chromium 实例.
"""
logger.info("开始连接 DrissionPage 浏览器,port={}", port)
browser = Chromium(port)
logger.info("DrissionPage 浏览器连接完成,port={}", port)
return browser
def start_client(self):
"""启动紫鸟客户端并更新浏览器内核."""
if self._is_client_ready():
logger.info("端口 {} 已启动,跳过启动紫鸟客户端", self.socket_port)
return
logger.info("端口 {} 未启动,开始启动紫鸟客户端", self.socket_port)
self.kill_process("v6")
time.sleep(CLIENT_RESTART_DELAY)
client_path = self.get_zinaio_exe("superbrowserv6")
if not client_path:
raise RuntimeError("未找到紫鸟客户端路径,无法启动客户端")
self.client_path = client_path.strip('"')
cmd = [
self.client_path,
"--run_type=web_driver",
"--ipc_type=http",
"--port=" + str(self.socket_port),
]
logger.info("紫鸟客户端启动命令:{}", " ".join(cmd))
for retry_count in range(CLIENT_START_RETRIES):
logger.info("{}/{} 次尝试启动紫鸟客户端", retry_count + 1, CLIENT_START_RETRIES)
process = subprocess.Popen(cmd)
if self._wait_until_client_ready(CLIENT_READY_TIMEOUT):
logger.info("紫鸟客户端启动成功,第 {} 次尝试", retry_count + 1)
time.sleep(CLIENT_POST_START_DELAY)
self.update_core()
return
logger.warning("{} 次尝试启动失败,10秒内未检测到客户端启动", retry_count + 1)
if process.poll() is not None:
logger.warning("紫鸟客户端进程已退出,returncode={}", process.returncode)
self._log_ziniao_webdriver_tail()
logger.error("紫鸟客户端启动失败,已重试 {}", CLIENT_START_RETRIES)
raise RuntimeError(f"客户端启动失败:重试 {CLIENT_START_RETRIES} 次后仍未成功启动")
def open_shop(self, shop_name: str):
"""打开指定店铺并完成 IP 检测.
Args:
shop_name: 店铺名称.
Returns:
成功时返回浏览器实例.店铺不存在时返回错误信息.
"""
logger.info("开始打开店铺:{}", shop_name)
self.start_client()
self.store_id = self._find_store_oauth(self.get_browser_list(), shop_name)
if not self.store_id:
logger.warning("店铺不存在:{}", shop_name)
return "店铺不存在"
ret_json = self.open_store(self.store_id)
self.store_id = ret_json.get("browserOauth")
if self.store_id is None:
self.store_id = ret_json.get("browserId")
self.browser = self.get_browser(ret_json.get("debuggingPort"))
ip_check_url = ret_json.get("ipDetectionPage")
if not ip_check_url:
logger.error("ip检测页地址为空,准备关闭店铺:{}", shop_name)
self.close_store(self.store_id)
raise RuntimeError("没有IP检测地址,为了店铺安全不打开店铺")
if self.open_ip_check(self.browser, ip_check_url):
logger.info("IP检测通过,打开店铺平台主页:{}", shop_name)
self.open_launcher_page(ret_json.get("launcherPage"), self.browser)
return self.browser
logger.error("IP检测不通过,停止打开店铺:{}", shop_name)
raise RuntimeError("IP检测不通过,可能是因为网络环境变化导致的,为了店铺安全不打开店铺")
def close_store(self, browser_oauth=None):
"""关闭紫鸟店铺浏览器.
Args:
browser_oauth: 店铺 OAuth 标识. 未提供时使用当前打开的店铺.
Returns:
紫鸟客户端返回的关闭结果.
"""
if browser_oauth is None:
browser_oauth = self.store_id
payload = self._build_payload(
"stopBrowser",
{
"duplicate": 0,
"browserOauth": browser_oauth,
},
)
request_id = payload["requestId"]
logger.info("开始关闭紫鸟店铺,browserOauth={},requestId={}", browser_oauth, request_id)
result = self._post_client(payload)
status_code = self._status_code(result)
if status_code == STATUS_OK:
logger.info("紫鸟店铺关闭成功,browserOauth={}", browser_oauth)
return result
if status_code == STATUS_LOGIN_FAILED:
logger.error("关闭紫鸟店铺登录失败:{}", self._result_text(result))
raise RuntimeError(f"【close_store】登录失败 {self._result_text(result)}")
logger.error("关闭紫鸟店铺失败:{}", self._result_text(result))
raise RuntimeError(f"[close_store]失败: {self._result_text(result)} ")
def open_launcher_page(self, launcher_page: str, browser: Chromium = None):
"""打开店铺平台启动页.
Args:
launcher_page: 要打开的启动页 URL.
browser: 浏览器实例. 未提供时使用当前浏览器实例.
"""
if browser is None:
browser = self.browser
logger.info("打开店铺平台启动页:{}", launcher_page)
tab = browser.new_tab(url=launcher_page)
self.tab = tab
return tab
def open_ip_check(self, browser: Chromium, ip_check_url: str):
"""打开 IP 检测页并判断网络环境是否通过.
Args:
browser: DrissionPage 浏览器会话.
ip_check_url: IP 检测页地址.
Returns:
IP 检测通过时返回 True,否则返回 False.
"""
try:
logger.info("开始打开IP检测页:{}", ip_check_url)
tab = browser.latest_tab
tab.get(ip_check_url)
success_button = tab.ele(
(By.XPATH, '//button[contains(@class, "styles_btn--success")]'),
timeout=IP_CHECK_TIMEOUT,
)
if success_button:
logger.info("IP检测成功")
return True
logger.warning("IP检测超时或未找到成功按钮")
return False
except Exception:
logger.exception("IP检测异常")
return False
class AmamzonBase(ZiniaoDriver):
"""亚马逊页面操作基类."""
def switch_to_country(self, country_name: str):
"""切换亚马逊账号国家并验证结果.
流程包括读取当前国家,打开国家下拉框,选择目标国家并等待页面加载.
Args:
country_name: 目标国家名称.
Returns:
切换成功返回 True,否则返回 False.
"""
try:
if self.browser is None:
logger.warning("浏览器实例不存在,请先打开店铺")
return False
self.tab.wait.doc_loaded(timeout=COUNTRY_INITIAL_LOAD_TIMEOUT, raise_err=False)
current_country = self._get_current_country(timeout=COUNTRY_LOOKUP_TIMEOUT)
if current_country is None:
logger.warning("无法获取当前国家信息")
return False
if current_country == country_name:
logger.info("当前已经是目标国家 {},无需切换", country_name)
return True
if not self._open_country_dropdown():
return False
if not self._select_country(country_name):
return False
logger.info("等待国家切换后页面加载")
self.tab.wait.doc_loaded()
return self._verify_country(country_name)
except Exception:
logger.exception("切换国家时发生异常")
return False
def _get_current_country(self, *, timeout: int) -> Optional[str]:
logger.info("正在检查当前国家")
current_country_ele = self.tab.ele(COUNTRY_LABEL_XPATH, timeout=timeout)
if not current_country_ele:
return None
current_country = current_country_ele.text.strip()
logger.info("当前国家:{}", current_country)
return current_country
def _open_country_dropdown(self) -> bool:
logger.info("正在打开国家切换下拉框")
dropdown_header = self.tab.ele(COUNTRY_DROPDOWN_XPATH, timeout=COUNTRY_CLICK_TIMEOUT)
if not dropdown_header:
logger.warning("找不到国家切换下拉框")
return False
logger.info("正在展开国家列表")
first_item = self._wait_country_list_item(dropdown_header)
if first_item is None:
logger.warning("找不到国家列表项")
return False
first_item.click()
time.sleep(COUNTRY_DROPDOWN_DELAY)
return True
def _wait_country_list_item(self, dropdown_header):
deadline = time.time() + COUNTRY_DROPDOWN_READY_TIMEOUT
attempt = 0
while time.time() < deadline:
attempt += 1
try:
dropdown_header.click()
except Exception as exc:
logger.info("点击国家切换下拉框失败, attempt={}, error={}", attempt, exc)
time.sleep(COUNTRY_DROPDOWN_DELAY)
first_item = self.tab.ele(COUNTRY_LIST_ITEM_XPATH, timeout=COUNTRY_DROPDOWN_RETRY_INTERVAL)
if first_item:
return first_item
self.tab.wait.doc_loaded(timeout=COUNTRY_CLICK_TIMEOUT, raise_err=False)
time.sleep(COUNTRY_DROPDOWN_RETRY_INTERVAL)
return None
def _select_country(self, country_name: str) -> bool:
logger.info("正在切换到国家:{}", country_name)
target_country = self.tab.ele(self._country_option_xpath(country_name), timeout=COUNTRY_CLICK_TIMEOUT)
if not target_country:
logger.warning("找不到目标国家:{}", country_name)
return False
target_country.click()
return True
@staticmethod
def _country_option_xpath(country_name: str) -> str:
return (
"xpath://div[@class="
'"dropdown-account-switcher-list-item dropdown-account-switcher-list-item-indented" '
f'and @title="{country_name}"]'
)
def _verify_country(self, country_name: str) -> bool:
new_country = self._get_current_country(timeout=COUNTRY_CLICK_TIMEOUT)
if new_country is None:
logger.warning("无法验证国家切换结果")
return False
if new_country == country_name:
logger.info("国家切换成功:{}", new_country)
return True
logger.warning("国家切换失败,当前国家:{},目标国家:{}", new_country, country_name)
return False
def need_login(self):
"""判断当前页面是否需要登录."""
time.sleep(NEED_LOGIN_DELAY)
self.tab.wait.doc_loaded(timeout=DOC_LOAD_TIMEOUT, raise_err=False)
need_login_ele = self.tab.eles(NEED_LOGIN_XPATH, timeout=NEED_LOGIN_TIMEOUT)
if len(need_login_ele) > 0:
logger.info("检测到需要登录元素")
return True
logger.info("未检测到登录元素")
return False
def login(self, password, username=""):
"""登录当前页面,必要时等待并提交一次性验证码.
Args:
password: 登录密码.
username: 兼容保留参数,当前未使用.
Returns:
登录成功返回 True,失败返回 False.
"""
try:
self.tab.wait.doc_loaded(timeout=DOC_LOAD_TIMEOUT, raise_err=False)
for _ in range(LOGIN_ATTEMPTS):
self._submit_password_if_present(password)
self.tab.wait.doc_loaded(timeout=DOC_LOAD_TIMEOUT, raise_err=False)
if self._send_otp_if_present():
continue
if self._submit_ready_otp():
return True
self._click_final_login_submit_if_present()
except Exception:
logger.exception("登录过程中发生异常")
return False
def _submit_password_if_present(self, password) -> None:
pwd_input = self.tab.eles(PASSWORD_INPUT_XPATH, timeout=PASSWORD_INPUT_TIMEOUT)
if len(pwd_input) > 0:
logger.info("检测到密码输入框,准备输入密码")
pwd_input[0].input(password, clear=True)
submit_btn = self.tab.eles(PASSWORD_SUBMIT_XPATH, timeout=PASSWORD_SUBMIT_TIMEOUT)
if len(submit_btn) > 0:
logger.info("点击登录提交按钮")
submit_btn[0].click()
def _send_otp_if_present(self) -> bool:
send_code = self.tab.eles(OTP_SEND_XPATH, timeout=OTP_SEND_TIMEOUT)
logger.info("发送一次性密码元素数量:{}", len(send_code))
if len(send_code) == 0:
return False
logger.info("检测到发送一次性密码按钮,准备点击")
send_code[0].click()
time.sleep(OTP_SEND_DELAY)
self.tab.wait.doc_loaded(timeout=DOC_LOAD_TIMEOUT, raise_err=False)
return True
def _submit_ready_otp(self) -> bool:
for _ in range(OTP_INPUT_LOOKUP_ATTEMPTS):
otp_input = self._find_otp_input()
if otp_input is None:
return False
if self._wait_for_otp_value(otp_input) and self._submit_otp_if_ready():
return True
return False
def _find_otp_input(self):
otp_code_input = self.tab.eles(OTP_INPUT_XPATH, timeout=OTP_INPUT_TIMEOUT)
logger.info("验证码输入框数量:{}", len(otp_code_input))
if len(otp_code_input) == 0:
return None
otp_code_input[0].wait.displayed(timeout=OTP_INPUT_DISPLAY_TIMEOUT, raise_err=False)
return otp_code_input[0]
@staticmethod
def _has_otp_value(otp_input) -> bool:
return otp_input.value is not None and otp_input.value.strip() != ""
def _wait_for_otp_value(self, otp_input) -> bool:
for _ in range(OTP_CODE_WAIT_SECONDS):
if self._has_otp_value(otp_input):
logger.info("检测到验证码输入完成")
return True
time.sleep(1)
return False
def _submit_otp_if_ready(self) -> bool:
submit_btn = self.tab.eles(OTP_SUBMIT_XPATH, timeout=OTP_SUBMIT_TIMEOUT)
if len(submit_btn) == 0:
return False
submit_btn[0].click()
self.tab.wait.doc_loaded(timeout=OTP_RESULT_LOAD_TIMEOUT, raise_err=False)
error_mes = self.tab.eles(OTP_ERROR_XPATH, timeout=OTP_ERROR_TIMEOUT)
if len(error_mes) > 0:
logger.warning("验证码输入错误:{}", error_mes[0].text)
self.tab.refresh()
return False
logger.info("登录成功")
return True
def _click_final_login_submit_if_present(self) -> None:
submit_btn = self.tab.eles(OTP_SUBMIT_XPATH, timeout=LOGIN_FINAL_SUBMIT_TIMEOUT)
if len(submit_btn) > 0:
logger.info("点击二次登录提交按钮")
submit_btn[0].click()

View File

@@ -1,195 +0,0 @@
import json
import sys
import os
import io
import time
import traceback
from datetime import datetime
from DrissionPage import Chromium, ChromiumOptions
from collections import defaultdict
from config import base_dir
from amazon.tool import get_shop_info,show_notification
class ChromeAmzoneBase:
mark_name = "谷歌浏览器基类"
country_info = {
"英国": {
"url": "https://www.amazon.co.uk/dp/B0CJ8SNXXV",
"zip_code": "SW1A 1AA",
"mark": "SW1A 1"
},
"德国": {
"url": "https://www.amazon.de/dp/B0CC8CW9G2?th=1",
"zip_code": "10115"
},
"法国": {
"url": "https://www.amazon.fr/dp/B0FRG1MJ8H?th=1",
"zip_code": "75001"
},
"西班牙": {
"url": "https://www.amazon.es/dp/B08ZXVNYNN",
"zip_code": "28001"
},
"意大利": {
"url": "https://www.amazon.it/dp/B0D1P17T2Q",
"zip_code": "20121"
}
}
def __init__(self):
"""
杀死当前谷歌浏览器进程,并使用 drissionpage 启动谷歌浏览器,使用系统安装的浏览器默认用户文件夹
"""
print("正在关闭现有的chromium浏览器进程...")
os.system('taskkill /f /t /im chrome.exe')
time.sleep(2)
print("正在启动chromium浏览器...")
# 配置浏览器选项
co = ChromiumOptions()
co.no_imgs(True)
user_data_path = os.path.join(base_dir, "user_data", "chrome_data")
if not os.path.exists(user_data_path):
os.makedirs(user_data_path, exist_ok=True)
co.set_user_data_path(user_data_path)
co.set_local_port(port=19897)
self.browser = Chromium(co)
self.tab = self.browser.latest_tab
print("Chrome浏览器启动成功")
def log(self, message: str, level: str = "INFO"):
"""日志输出
Args:
message: 日志消息
level: 日志级别
"""
try:
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if level == "ERROR":
show_notification(message, "error")
print(f"[{timestamp}] [{self.mark_name}] [{level}] {message}")
except Exception as e:
print(f"输出出错,{e}")
def close_init_popup(self):
"""
关闭所有的初始化弹窗
"""
self.tab.wait.doc_loaded(timeout=60, raise_err=True)
continue_shipping = self.tab.eles('xpath://button[@alt="Continue shopping"]',timeout=3)
if len(continue_shipping) > 0:
continue_shipping[0].click()
time.sleep(0.5)
self.tab.wait.doc_loaded(timeout=60, raise_err=True)
footbar = self.tab.eles('xpath://footer[@class="el-dialog__footer"]', timeout=5)
if len(footbar) > 0:
do_not_remind = footbar[0].eles('xpath:.//input[@class="el-checkbox__original"]')
if len(do_not_remind) > 0:
do_not_remind[0].check()
resume_immediately = footbar[0].eles('xpath:.//button')
if len(resume_immediately) > 0:
resume_immediately[0].click()
self.tab.wait.doc_loaded(timeout=60, raise_err=True)
accept_btn = self.tab.eles('xpath://input[@id="sp-cc-accept"]', timeout=5)
if len(accept_btn) > 0:
accept_btn[0].click()
def _set_zip_code(self, zip_code, mark=None):
"""
设置邮编
Args:
zip_code: 目标邮编
"""
try:
# 检查当前邮编
zip_display = self.tab.ele('xpath://div[@id="glow-ingress-block"]', timeout=10)
if zip_display:
current_text = zip_display.text
# print(f"当前地址信息: {current_text}")
# 先检查标识
if mark is not None and mark in current_text:
print(f"邮编检测到标识: {mark},无需修改")
return True
# 检查是否已经包含目标邮编
if zip_code in current_text:
print(f"邮编已经设置为: {zip_code},无需修改")
return True
# 需要设置邮编
print(f"正在设置邮编为: {zip_code}")
# 点击地址选择按钮
location_link = self.tab.ele('xpath://a[@id="nav-global-location-popover-link"]', timeout=10)
if not location_link:
print("找不到地址设置按钮")
return False
location_link.click()
time.sleep(1)
# 等待邮编输入框出现
zip_input = self.tab.ele('xpath://input[@id="GLUXZipUpdateInput"]', timeout=10)
if not zip_input:
print("找不到邮编输入框")
return False
# 输入邮编
zip_input.input(zip_code, clear=True)
time.sleep(0.5)
# 点击提交按钮
submit_btn = self.tab.ele('xpath://input[@aria-labelledby="GLUXZipUpdate-announce"]', timeout=10)
if not submit_btn:
print("找不到提交按钮")
return False
submit_btn.click()
continue_btn = self.tab.eles('xpath://div[@class="a-popover-footer"]//input[@id="GLUXConfirmClose"]',
timeout=10)
if len(continue_btn) > 0:
continue_btn[0].click()
# 等待提交按钮消失(表示请求已发送)
print("等待邮编更新...")
time.sleep(2)
# 等待页面加载完成
self.tab.wait.doc_loaded(timeout=30, raise_err=False)
time.sleep(2)
# 验证邮编是否设置成功
zip_display_after = self.tab.ele('xpath://div[@id="glow-ingress-block"]', timeout=10)
if zip_display_after:
updated_text = zip_display_after.text
# print(f"更新后的地址信息: {updated_text}")
if zip_code in updated_text:
print(f"邮编设置成功: {zip_code}")
return True
else:
# print(f"邮编设置可能失败,当前显示: {updated_text}")
return False
return True
except Exception as e:
print(f"设置邮编时出错: {traceback.format_exc()}")
return False
def close(self):
"""关闭浏览器"""
try:
if self.browser:
self.browser.quit()
print("浏览器已关闭")
except Exception as e:
print(f"关闭浏览器时出错: {str(e)}")

View File

@@ -1,551 +0,0 @@
import traceback
import winreg
import subprocess
import time
import uuid
import requests
import json
import os
from datetime import datetime
from amazon.amazon_base import AmamzonBase,kill_process,TaskBase
from amazon.tool import show_notification,get_shop_info
from config import runing_task,runing_shop,DELETE_BRAND_API_BASE
class AmazoneDriver(AmamzonBase):
"""亚马逊专用驱动类继承自ZiniaoDriver"""
mark_name = "删除品牌"
def SwitchPage(self):
"""
切换至 管理所有库存页面
1、等待 //navigation-favorites-bar[@class="hydrated"] 出现
"""
navigation = self.tab.ele('xpath://navigation-favorites-bar[@class="hydrated"]')
navigation.wait.displayed(raise_err=False)
page_btn = navigation.sr('xpath://internal-fav-bar-links[@data-internal="navigation"]').sr(
'xpath://a[@data-page-id="ezdpc-gui-inventory-mons"]')
page_btn.wait.displayed(raise_err=False)
page_btn.click(timeout=5)
self.tab.wait.doc_loaded()
# 等待搜索框出现
search_region = self.tab.ele('xpath://div[@id="searchBoxContainer"]//kat-input-group')
search_region.wait.displayed(raise_err=False)
def search(self, asin):
search_region = self.tab.ele('xpath://div[@id="searchBoxContainer"]//kat-input-group')
search_region.wait.displayed(raise_err=False)
time.sleep(0.6)
search_input = self.tab.ele("xpath://kat-input[contains(@class,'SearchBox-module__searchInput')]").sr(
'xpath://span[@class="container"]//input[@part="input"]')
search_input.input(asin,clear=True)
sku_ls = []
for _ in range(3):
search_btn = self.tab.ele("xpath://kat-icon[@name='search']")
search_btn.click()
load_ele = self.tab.ele("xpath://div[contains(@class,'Loader-module__loader')]")
# load_ele.wait.hidden(timeout=3, raise_err=False)
load_ele.wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=3)
if len(sku_ls) > 0:
break
return sku_ls
def del_action(self, sku_ele):
dropdown = sku_ele.ele('xpath:.//kat-dropdown-button[@variant="secondary"]')
dropdown.click()
time.sleep(1)
del_btn = dropdown.sr("xpath:.//button[@role='menuitem' and @data-action='DeleteListing']")
del_btn.wait.displayed(raise_err=False)
del_btn.click()
confirm_btn = self.tab.ele('xpath://kat-modal[@data-testid="action-modal"]//kat-button[@variant="primary"]')
confirm_btn.wait.displayed(raise_err=False)
confirm_btn.wait.enabled()
confirm_btn.click()
suc_alert = self.tab.ele("xpath://kat-alert[@variant='success']")
suc = suc_alert.wait.displayed(raise_err=False, timeout=5)
return suc
class DelbramdTask(TaskBase):
task_name = "删除品牌-TASK"
def update_task_status(self, task_id: int, **kwargs):
"""更新任务状态
Args:
task_id: 任务ID
**kwargs: 要更新的字段
"""
if task_id not in runing_task:
runing_task[task_id] = {}
runing_task[task_id].update(kwargs)
def process_task(self,task_data):
try:
data = task_data.get("data", {})
task_id = data.get("taskId")
items = data.get("items", [])
if not task_id:
self.log("任务ID为空跳过", "WARNING")
return
# 初始化任务状态
self.update_task_status(
task_id,
status="running",
start_time=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
total_shops=len(items),
processed_shops=0,
total_asins=sum(shop.get("totalRows", 0) for shop in items),
processed_asins=0,
success_count=0,
failed_count=0,
stop_requested=False # 暂停请求标志
)
self.log(f"开始处理任务 {task_id},共 {len(items)} 个店铺")
# 遍历处理每个店铺
for idx, shop_data in enumerate(items, 1):
shop_name = shop_data.get("shopName", "未知店铺")
self.log(f"[{idx}/{len(items)}] 开始处理店铺: {shop_name}")
try:
self.process_shop(shop_data, task_id)
# 更新已处理店铺数
if task_id in runing_task:
runing_task[task_id]["processed_shops"] += 1
except Exception as e:
self.log(f"处理店铺 {shop_name} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 检查任务最终状态
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.update_task_status(task_id, status="stopped")
self.log(f"任务 {task_id} 已被暂停!")
else:
self.update_task_status(task_id, status="completed")
self.log(f"任务 {task_id} 处理完成!")
except Exception as e:
self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR")
if task_id:
self.update_task_status(task_id, status="failed", error=str(e))
def process_shop(self, shop_data, task_id):
"""处理单个店铺(包含重试逻辑)
Args:
shop_data: 店铺数据
task_id: 任务ID
"""
shop_name = shop_data.get("shopName", "未知店铺")
result_id = shop_data.get("resultId")
countries = shop_data.get("countries", [])
company_name = shop_data.get("companyName", "未知公司")
if not countries:
self.log(f"店铺 {shop_name} 没有国家数据,跳过", "WARNING")
return
# 更新当前处理的店铺
self.update_task_status(task_id, current_shop=shop_name)
# 将店铺添加到正在执行中的店铺列表
start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
runing_shop[shop_name] = start_time
self.log(f"账号:{company_name},店铺 {shop_name} 已添加到执行列表,开始时间: {start_time}")
# 店铺打开重试最多3次
max_retries = 5
iskill = False
# 处理每个国家
chunk_index = 1
for country_data in countries:
self.log(f"开始处理国家:{country_data}")
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理国家", "WARNING")
break # 跳出循环进入finally关闭店铺
# 打开店铺
driver = self.open_shop(AmazoneDriver,max_retries=max_retries, company_name=company_name,
shop_name=shop_name, iskill=iskill)
# 检查是否成功打开
if not driver:
self.log(f"店铺 {shop_name} 打开失败,已重试 {max_retries} 次,跳过该店铺", "ERROR")
return
try:
chunk_index = self.process_country(driver, country_data, task_id, result_id, shop_data, chunk_index)
except Exception as e:
country_name = country_data.get("country", "未知")
self.log(f"处理国家 {country_name} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
self.log(traceback.format_exc(), "ERROR")
if "与页面的连接已断开" in str(e):
iskill = True
try:
driver.close_store()
except Exception as e:
iskill = True
self.log(f"关闭国家失败,{e}")
# 从正在执行中的店铺列表中移除
if shop_name in runing_shop:
del runing_shop[shop_name]
self.log(f"店铺 {shop_name} 已从执行列表中移除")
def process_country(self, driver: AmazoneDriver, country_data, task_id: int,
result_id: int, shop_data, chunk_index: int):
"""处理单个国家的所有ASIN
Args:
driver: 亚马逊驱动实例
country_data: 国家数据
task_id: 任务ID
result_id: 结果ID
shop_data: 店铺数据
"""
country = country_data.get("country", "未知")
items = country_data.get("items", [])
shop_name = shop_data.get("shopName", "未知店铺")
self.log(f"开始处理国家: {country},共 {len(items)} 个ASIN")
self.update_task_status(task_id, current_country=country)
max_retries = 5
switch_success, switch_success_pg, search_success, sku_ls = self.action_init(driver, country, shop_name,
None)
# 如果切换失败,直接返回
if not switch_success or not switch_success_pg or not search_success:
error_message = f"切换到国家({switch_success})/库存页面({switch_success_pg})/搜索筛选到指定选项({search_success}) {country} 失败,已重试 {max_retries} 次,跳过该国家"
self.log(error_message, "ERROR")
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
return chunk_index
# 处理每个ASIN
file_key = shop_data.get("fileKey", "")
source_filename = shop_data.get("sourceFilename", "")
total_rows = shop_data.get("totalRows", 0)
for idx, asin_item in enumerate(items, 1):
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理", "WARNING")
return # 返回到上层,会触发店铺关闭
try:
self.log(f"[{idx}/{len(items)}] 处理ASIN: {asin_item.get('asin', '')}")
self.process_asin(driver, asin_item, country, task_id, result_id,
file_key, source_filename, total_rows, chunk_index)
except Exception as e:
asin = asin_item.get("asin", "未知")
self.log(f"处理ASIN {asin} 失败: {str(e)}", "ERROR")
# 继续处理下一个ASIN
chunk_index += 1
return chunk_index
def process_asin(self, driver: AmazoneDriver, asin_item,
country: str, task_id: int, result_id: int, file_key: str,
source_filename: str, total_rows: int, chunk_index: int):
"""处理单个ASIN并回传结果
Args:
driver: 亚马逊驱动实例
asin_item: ASIN数据项
country: 国家名称
task_id: 任务ID
result_id: 结果ID
file_key: 文件KEY
source_filename: 源文件名
total_rows: 总行数
chunk_index: 当前处理索引
"""
asin = asin_item.get("asin", "")
row_index = asin_item.get("rowIndex", 0)
# 更新当前处理的ASIN
self.update_task_status(task_id, current_asin=asin)
status = "失败"
max_retries = 3 # 最多重试3次
for retry in range(max_retries):
try:
self.log(f"处理ASIN {asin} (第 {retry + 1}/{max_retries} 次)")
# 如果不是第一次尝试,先刷新页面
if retry > 0:
self.log("重试前刷新页面...")
try:
driver.tab.refresh()
time.sleep(3)
except Exception as e:
self.log(f"刷新页面失败: {str(e)}", "WARNING")
# 搜索ASIN
sku_ls = driver.search(asin=asin)
self.log(f"搜索到 {len(sku_ls)} 个SKU")
if len(sku_ls) == 0:
status = "查询不到"
self.log(f"ASIN {asin} 未找到商品", "WARNING")
break # 查询不到商品,无需重试
else:
# 删除所有找到的SKU如果任何一个失败则重新开始整个流程
success_count = 0
all_success = True # 标记是否所有SKU都删除成功
total_sku_count = len(sku_ls)
for sku in sku_ls:
try:
suc = driver.del_action(sku)
if suc:
success_count += 1
self.log(f"SKU 删除成功 ({success_count}/{total_sku_count})")
else:
self.log(f"SKU 删除失败", "WARNING")
all_success = False
break # 任何一个失败,退出循环,准备重试整个流程
except Exception as e:
self.log(f"删除SKU异常: {str(e)}", "ERROR")
all_success = False
break # 发生异常,退出循环,准备重试整个流程
# 判断删除结果
if all_success and success_count == total_sku_count:
status = "成功"
self.log(f"ASIN {asin} 所有SKU删除成功 ({success_count}/{total_sku_count})")
# 更新成功计数
if task_id in runing_task:
runing_task[task_id]["success_count"] += 1
break # 全部成功,跳出重试循环
else:
# 有失败的SKU
if retry < max_retries - 1:
self.log(f"有SKU删除失败准备重试整个流程... ({retry + 1}/{max_retries})")
time.sleep(2)
continue # 继续下一次重试
else:
# 所有重试都用完了
if success_count > 0:
status = "部分成功"
self.log(f"ASIN {asin} 部分SKU删除成功 ({success_count}/{total_sku_count})", "WARNING")
if task_id in runing_task:
runing_task[task_id]["success_count"] += 1
else:
status = "失败"
self.log(f"ASIN {asin} 所有SKU删除失败", "ERROR")
if task_id in runing_task:
runing_task[task_id]["failed_count"] += 1
except Exception as e:
status = "删除异常"
self.log(f"处理ASIN {asin} 异常: {str(e)}", "ERROR")
# 如果还有重试机会,继续重试
if retry < max_retries - 1:
self.log(f"发生异常,准备重试... ({retry + 1}/{max_retries})")
time.sleep(2)
else:
# 所有重试都失败了,更新失败计数
if task_id in runing_task:
runing_task[task_id]["failed_count"] += 1
# 更新已处理ASIN计数
if task_id in runing_task:
runing_task[task_id]["processed_asins"] += 1
# 回传结果到API
try:
payload = {
"submissionId": "",
"files": [{
"fileKey": file_key,
"sourceFilename": source_filename,
"chunkIndex": chunk_index,
"chunkTotal": total_rows,
"processedRows": chunk_index,
"totalRows": total_rows,
"currentCountry": country,
"currentAsin": asin,
"countries": [{
"country": country,
"items": [{
"asin": asin,
"status": status
}]
}]
}]
}
self.post_result(task_id, payload)
self.log(f"ASIN {asin} 结果已回传,状态: {status}")
except Exception as e:
self.log(f"回传结果失败: {str(e)}", "ERROR")
def _report_all_asins_failed(self, country: str, items, task_id, shop_data, chunk_index):
"""将国家下所有ASIN标记为失败并回传
Args:
country: 国家名称
items: ASIN列表
task_id: 任务ID
shop_data: 店铺数据
"""
self.log(f"开始回传国家 {country} 下的 {len(items)} 个ASIN失败状态")
file_key = shop_data.get("fileKey", "")
source_filename = shop_data.get("sourceFilename", "")
total_rows = shop_data.get("totalRows", 0)
for asin_item in items:
asin = asin_item.get("asin", "")
try:
# 更新失败计数
if task_id in runing_task:
runing_task[task_id]["failed_count"] += 1
runing_task[task_id]["processed_asins"] += 1
# 回传失败状态
payload = {
"submissionId": "",
"files": [{
"fileKey": file_key,
"sourceFilename": source_filename,
"chunkIndex": chunk_index,
"chunkTotal": total_rows,
"processedRows": chunk_index,
"totalRows": total_rows,
"currentCountry": country,
"currentAsin": asin,
"countries": [{
"country": country,
"items": [{
"asin": asin,
"status": "失败"
}]
}]
}]
}
self.post_result(task_id, payload)
chunk_index += 1
self.log(f"ASIN {asin} 失败状态已回传")
except Exception as e:
self.log(f"回传ASIN {asin} 失败状态时出错: {str(e)}", "ERROR")
return chunk_index
def post_result(self, task_id: int, payload):
"""回传结果到API带重试机制
Args:
task_id: 任务ID
payload: 结果数据
"""
url = f"{DELETE_BRAND_API_BASE}/api/delete-brand/tasks/{task_id}/result"
max_retries = 3 # 最多重试3次
for retry in range(max_retries):
try:
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
response = requests.post(
url,
json=payload,
headers={"Content-Type": "application/json"},
timeout=30,
verify=False # 忽略SSL证书验证
)
print("【结果提交】:", payload)
print("【结果提交返回】:", response.text)
if response.status_code == 200:
self.log(f"结果回传成功: {url}")
return # 成功后直接返回,不再重试
else:
self.log(f"结果回传失败,状态码: {response.status_code}", "WARNING")
# 如果还有重试机会,继续重试
if retry < max_retries - 1:
self.log(f"准备重试... ({retry + 1}/{max_retries})")
time.sleep(2) # 等待2秒后重试
else:
self.log(f"已达到最大重试次数,结果回传最终失败", "ERROR")
except Exception as e:
self.log(f"调用API异常: {str(e)}", "ERROR")
# 如果还有重试机会,继续重试
if retry < max_retries - 1:
self.log(f"发生异常,准备重试... ({retry + 1}/{max_retries})")
time.sleep(2) # 等待2秒后重试
else:
self.log(f"已达到最大重试次数,结果回传最终失败", "ERROR")
def update_task_status(self, task_id: int, **kwargs):
"""更新任务状态
Args:
task_id: 任务ID
**kwargs: 要更新的字段
"""
if task_id not in runing_task:
runing_task[task_id] = {}
runing_task[task_id].update(kwargs)
if __name__ == '__main__':
# 使用示例
user_info = {
"company": "rongchuang123",
"username": "自动化_Robot",
"password": "#20zsg25"
}
shop_name = "郭亚芳"
kill_process("v6")
# 创建驱动实例
driver = AmazoneDriver(user_info)
# 打开店铺并获取浏览器实例
browser = driver.open_shop(shop_name)
country = "西班牙"
asin = "Voanos"
sw_suc = driver.SwitchingCountries(country)
driver.SwitchPage()
sku_ls = driver.search(asin=asin)
print("查询结果有:", len(sku_ls))
for sku in sku_ls:
suc = driver.del_action(sku)
print(sku, "删除结果", suc)

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,175 +0,0 @@
import traceback
import requests
import threading
from datetime import datetime
from typing import Dict, Any, List
from concurrent.futures import ThreadPoolExecutor, as_completed
from config import JSON_TASK_QUEUE, runing_task, runing_shop, DELETE_BRAND_API_BASE, ZN_COMPANY, ZN_USERNAME, ZN_PASSWORD
from amazon.del_brand import AmazoneDriver, DelbramdTask
from amazon.approve import ApproveTask
from amazon.match_action import MatchTak
from amazon.price_match import PriceTask
from amazon.asin_status import StatusTask
from amazon.patrol_delete import PatrolDeleteTask
from amazon.detail_spider import SpiderTask
from amazon.similar_asin import SimilarAsinTask
from amazon.amazon_base import kill_process
class TaskMonitor:
"""任务监控器:负责监控队列并执行品牌删除任务"""
def __init__(self):
"""初始化任务监控器"""
self.running = True
self.user_info = {
"company": ZN_COMPANY,
"username": ZN_USERNAME,
"password": ZN_PASSWORD
}
self.chunk_index = 1 # 当前处理的分块索引
self.max_workers = 5 # 线程池最大线程数
self.executor = None # 线程池执行器
self.serial_task_locks = {
"product-risk-resolve-run": threading.Lock(),
}
# 在提交新任务前杀掉旧进程(确保环境干净)
kill_process("v6")
kill_process("v5")
def log(self, message: str, level: str = "INFO"):
"""日志输出
Args:
message: 日志消息
level: 日志级别
"""
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"[{timestamp}] [{level}] {message}")
def start(self):
"""启动任务监控(使用线程池处理任务)"""
self.log(f"任务监控器启动,开始监听队列... (线程池大小: {self.max_workers})")
# 创建线程池
self.executor = ThreadPoolExecutor(max_workers=self.max_workers)
futures = [] # 保存所有提交的任务Future对象
task_type_info = {
"delete-brand-run" : "删除品牌",
"product-risk-resolve-run" : "产品风险审批",
"shop-match-run" : "匹配价格",
"price-track-run" : "跟价",
"query-asin-run" : "状态查询",
"patrol-delete-run" : "巡店删除",
"appearance-patent-run" : "亚马逊采集",
"similar-asin-run" : "相似ASIN"
}
try:
while self.running:
try:
# 超时后继续循环检查self.running状态避免卡死
task_data = JSON_TASK_QUEUE.get(block=True, timeout=30)
# 检查任务类型
task_type = task_data.get("type", "")
if task_type in task_type_info:
# 提交产品风险审批任务到线程池
self.log(f"接收到任务,提交到线程池处理...")
future = self.executor.submit(self._process_approve_task_wrapper, task_data, task_type_info[task_type])
futures.append(future)
else:
self.log(f"未知任务类型: {task_type},跳过", "WARNING")
continue
# 清理已完成的future对象避免内存累积
futures = [f for f in futures if not f.done()]
except Exception as e:
# 静默处理队列为空的超时,只记录真正的异常
if str(e) and "Empty" not in str(e):
self.log(f"任务监控异常: {str(e)}", "ERROR")
# 队列为空时不需要额外等待,直接继续循环
finally:
# 关闭监控时,等待所有任务完成
self.log("正在关闭任务监控器,等待所有任务完成...")
if self.executor:
self.executor.shutdown(wait=True)
self.log("所有任务已完成,监控器已关闭")
def _process_task_wrapper(self, task_data: Dict[str, Any]):
"""任务处理包装器(用于线程池调用)
Args:
task_data: 任务数据
"""
try:
self.log(f"线程 {id(task_data)} 开始处理任务...")
self.process_task(task_data)
self.log(f"线程 {id(task_data)} 任务处理完成")
except Exception as e:
self.log(f"线程 {id(task_data)} 删除品牌任务处理异常: {traceback.format_exc()}", "ERROR")
def _process_approve_task_wrapper(self, task_data: Dict[str, Any],TASK_TYPE:str):
task_type = task_data.get("type", "")
task_lock = self.serial_task_locks.get(task_type)
if task_lock is not None:
self.log(f"线程 {id(task_data)} 等待 {task_type} 串行锁...")
with task_lock:
self.log(f"线程 {id(task_data)} 获取 {task_type} 串行锁,开始处理...")
return self._process_approve_task_unlocked(task_data, TASK_TYPE)
return self._process_approve_task_unlocked(task_data, TASK_TYPE)
def _process_approve_task_unlocked(self, task_data: Dict[str, Any],TASK_TYPE:str):
"""审批任务处理包装器(用于线程池调用)
Args:
task_data: 任务数据
"""
try:
TASK_INFO = {
"删除品牌": DelbramdTask ,
"产品风险审批" : ApproveTask,
"匹配价格" : MatchTak,
"跟价" : PriceTask,
"状态查询" : StatusTask,
"巡店删除" : PatrolDeleteTask,
"亚马逊采集" : SpiderTask,
"相似ASIN" : SimilarAsinTask
}
self.log(f"线程 {id(task_data)} 开始处理产品风险审批任务...")
# 创建ApproveTask实例并处理任务
TASK_CLS = TASK_INFO[TASK_TYPE] # 根据任务类型选择处理类默认为ApproveTask
approve_task = TASK_CLS(user_info=self.user_info)
approve_task.process_task(task_data)
self.log(f"线程 {id(task_data)} {TASK_TYPE} 任务处理完成")
except Exception as e:
self.log(f"线程 {id(task_data)} {TASK_TYPE} 任务处理异常: {traceback.format_exc()}", "ERROR")
def stop(self):
"""停止监控"""
self.running = False
self.log("任务监控器已停止")
def main():
"""主函数:启动任务监控"""
# 创建并启动任务监控器
monitor = TaskMonitor()
try:
monitor.start()
except KeyboardInterrupt:
monitor.log("接收到中断信号,正在停止...")
monitor.stop()
except Exception:
monitor.log(f"监控器异常退出: {traceback.format_exc()}", "ERROR")
if __name__ == "__main__":
main()

View File

@@ -1,559 +0,0 @@
import time
import re
import traceback
from config import runing_task, runing_shop,DELETE_BRAND_API_BASE
from datetime import datetime
from amazon.amazon_base import AmamzonBase, kill_process,TaskBase
from amazon.tool import show_notification
import requests
class AmzoneMatchAction(AmamzonBase):
mark_name = "匹配价格"
def __init__(self, user_info: dict, socket_port: int = 19890):
super().__init__(user_info,socket_port)
self.already_asin = set()
def reset_already_asin(self):
self.already_asin = set()
self.log("already_asin 已重置")
def wait_loaded(self):
# 等待加载完成
try:
load_ele = self.tab.ele(
'xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]//div[contains(@class,"Loader-module__loader")]|//kat-panel[@data-testid="kat-panel-ActionPanelContent"]/div[@data-f1-component]//div[contains(@class,"==")]/div[contains(@class,"==")]/span',
timeout=3)
load_ele.wait.deleted(timeout=5, raise_err=False)
except Exception as e:
print(f"{self.mark_name}】等待加载中消失出错", e)
def run_page_action(self):
self.log(f"==================={self.mark_name}=======================")
self.log("开始执行...")
num = 0
retry_num = 0
get_page_faild = 0
while retry_num < 3: # 最多重试3次
# if num > 3: #测试
# return
# 等待加载完成
try:
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]",timeout=5)
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
# 获取当前页码
try:
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
if len(page_pamel) > 0:
current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text
# 测试
# if current_page > 1:
# break
# 总页数
total_page = page_pamel[0].sr.eles('xpath:.//ul[@class="pages"]//span[@class="page__inner"][last()]')
if len(total_page) > 0:
total_page = total_page[-1].text
else:
total_page = 0
self.log(f"{self.mark_name}】当前页码: {current_page} / 总页数: {total_page}")
get_page_faild = 0
except Exception as e:
self.log(f"{self.mark_name}】获取页码失败", e)
get_page_faild+= 1
if get_page_faild > 2:
show_notification(f"{self.mark_name}】获取页码失败超3次停止任务")
break
# 保存当前URL用于失败重试时访问
try:
self.url = self.tab.url
self.log(f"获取当前的链接:{self.url}")
except Exception as e:
self.url = None
self.log(f"获取当前的链接失败:{e}")
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=10)
self.log(f"{self.mark_name}】获取到 {len(sku_ls)}")
# for sku_ele in sku_ls[0:2]:
for sku_ele in sku_ls:
# solve_problem = sku_ele.eles('xpath:.//kat-link[@label="解决商品信息问题"]')
asin = sku_ele.ele('xpath:.//div[contains(@class,"JanusSplitBox-module__container")]//div[contains(@class,"JanusSplitBox-module__panel--") and contains(string(.),"ASIN")]/..//div[last()]',timeout=3).text
self.log(f"{self.mark_name}】ASIN {asin} 找到....")
if asin in self.already_asin:
self.log(f"{self.mark_name}{asin} 已经处理过了,跳过")
continue
price_match = sku_ele.eles('xpath:.//div[@data-test-id="FeaturedOfferPrice"]//a[text()="匹配"]',timeout=2)
if len(price_match) == 0:
self.log(f"{self.mark_name}{asin},没有推荐价格匹配按钮")
yield (asin,"无需处理")
self.already_asin.add(asin)
continue
try:
price_match[0].click()
save_all_btn = self.tab.ele('xpath://kat-button[@label="保存所有"]',timeout=10)
save_all_btn.wait.displayed(timeout=10, raise_err=False)
save_all_btn.click()
save_all_btn.wait.deleted(timeout=10, raise_err=False)
yield (asin,"处理完成")
self.already_asin.add(asin)
except Exception as e:
print(f"{asin} 处理失败", e)
yield (asin,"处理失败")
self.already_asin.add(asin)
continue
# 判断是否存在需要翻页的情况
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
if len(page_pamel) == 0:
break
next_page_btn = page_pamel[0].sr('xpath:.//span[@part="pagination-nav-right"]')
class_str = next_page_btn.attr('class')
if "end" in class_str:
break
next_page_btn.click()
num += 1
self.log(f"{self.mark_name}】【程序计算】正在翻页,已翻 {num} 页...")
except Exception as e:
self.log(f"{self.mark_name}】处理匹配操作异常", e)
traceback.print_exc()
retry_num += 1
self.tab.refresh()
self.tab.wait.doc_loaded(raise_err=False,timeout=120)
class MatchTak(TaskBase):
task_name = "匹配价格-TASK"
def process_task(self, task_data: dict):
"""处理审批任务主入口
Args:
task_data: 任务数据
"""
try:
data = task_data.get("data", {})
task_id = data.get("taskId")
items = data.get("items", [])
country_codes = data.get("country_codes", [])
risk_listing_filter = data.get("risk_listing_filter", "All")
user_id = data.get("user_id")
stage_index = data.get("stage_index")
final_stage = bool(data.get("final_stage", True))
# 用于测试
limit = data.get("limit",None)
if not task_id or not items or not country_codes:
self.log(f"任务ID为空{task_id}/店铺列表为空{items}/国家列表为空{country_codes},跳过", "WARNING")
return
self.log(f"开始处理审批任务 {task_id},共 {len(items)} 个店铺,{len(country_codes)} 个国家")
runing_task[task_id] = {
"status": "running",
"start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"total_shops": len(items),
"processed_shops": 0,
"total_countries": len(country_codes) * len(items),
"processed_countries": 0,
"total_asins": 0,
"processed_asins": 0,
"success_count": 0,
"failed_count": 0,
"stop_requested": False
}
# 遍历处理每个店铺
for idx, shop_item in enumerate(items, 1):
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理", "WARNING")
runing_task[task_id]["status"] = "stopped"
return
shop_name = shop_item.get("shopName", "未知店铺")
self.log(f"[{idx}/{len(items)}] 开始处理店铺: {shop_name}")
show_notification(f"开始处理店铺: {shop_name}", "info")
try:
self.process_shop(shop_item, country_codes, task_id, risk_listing_filter, user_id, stage_index, final_stage,limit=limit)
# self.process_shop(shop_item, country_codes, task_id,risk_listing_filter)
# 更新已处理店铺数
if task_id in runing_task:
runing_task[task_id]["processed_shops"] += 1
except Exception as e:
self.log(f"处理店铺 {shop_name} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 更新任务状态
if task_id in runing_task:
if runing_task[task_id].get("stop_requested", False):
runing_task[task_id]["status"] = "stopped"
self.log(f"任务 {task_id} 已被暂停!")
else:
runing_task[task_id]["status"] = "completed"
self.log(f"任务 {task_id} 处理完成!")
except Exception as e:
self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR")
if task_id:
if task_id in runing_task:
runing_task[task_id]["status"] = "failed"
runing_task[task_id]["error"] = str(e)
def process_shop(self, shop_item: dict, country_codes: list, task_id: int, risk_listing_filter: str,
user_id=None, stage_index=None, final_stage: bool = True,limit:str=None):
"""处理单个店铺
Args:
shop_item: 店铺信息
country_codes: 国家代码列表
task_id: 任务ID
risk_listing_filter: 风险商品筛选条件
"""
shop_name = shop_item.get("shopName", "未知店铺")
company_name = shop_item.get("companyName", "")
if not company_name:
self.log(f"店铺 {shop_name} 的公司名称为空,跳过", "WARNING")
return
if task_id in runing_task:
runing_task[task_id]["current_shop"] = shop_name
# 将店铺添加到正在执行中的店铺列表
start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
runing_shop[shop_name] = start_time
self.log(f"店铺 {shop_name} 已添加到执行列表,账号: {company_name},开始时间: {start_time}")
# 店铺打开重试最多3次
driver = None
max_retries = 3
iskill = False
# 处理每个国家
for country_code in country_codes:
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理国家", "WARNING")
break
# 打开店铺
driver = self.open_shop(cls=AmzoneMatchAction,max_retries=max_retries, company_name=company_name,
shop_name=shop_name, iskill=iskill)
if driver is None:
self.log(f"任务 {task_id} 启动店铺失败,结束任务", "ERROR")
for country_code in country_codes:
self.post_result(task_id, shop_name, country_code, "", "", is_done=True)
return
current_url = None
for _ in range(max_retries):
try:
self.process_country(driver, country_code, task_id, shop_name,risk_listing_filter,limit,
target_url=current_url)
driver.reset_already_asin()
driver.close_store()
break
except Exception as e:
self.log(f"处理国家 {country_code} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
if "与页面的连接已断开" in str(e):
iskill = True
current_url = driver.url
# 更新已处理国家数
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
self.log(f"{task_id}任务处理完成")
# 最后回传,标记完成
try:
# self.post_result(task_id, shop_name, country_code, "", "", is_done=True)
if final_stage:
self.post_result(task_id, shop_name, country_code, "", "", is_done=True)
else:
self.post_stage_finished(task_id, user_id, stage_index)
except Exception as e:
self.log(f"回传结果失败: {str(e)}", "ERROR")
# 从正在执行中的店铺列表中移除
if shop_name in runing_shop:
del runing_shop[shop_name]
self.log(f"店铺 {shop_name} 已从执行列表中移除")
def process_country(self, driver, country_code, task_id, shop_name, risk_listing_filter,limit=None,target_url=None):
"""处理单个国家的审批任务
"""
# 转换国家代码为中文名称
country_name = self.country_info.get(country_code, country_code)
info_mes = f"开始处理国家: {country_name} ({country_code})"
self.log(info_mes)
show_notification(info_mes, "info")
# 更新当前处理的国家
if task_id in runing_task:
runing_task[task_id]["current_country"] = country_name
max_retries = 5
switch_success, switch_success_pg, search_success, sku_ls = self.action_init(driver, country_name, shop_name,
risk_listing_filter)
# 如果切换失败,直接返回
if not switch_success or not switch_success_pg or not search_success:
error_message = f"切换到国家({switch_success})/库存页面({switch_success_pg})/搜索筛选到指定选项({search_success}) {country_name} 失败,已重试 {max_retries} 次,跳过该国家"
self.log(error_message, "ERROR")
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
return
# # 如果没有需要审批的商品,直接返回
if len(sku_ls) == 0:
self.log(f"国家 {country_name} 没有需要审批的商品")
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
return
self.log(f"国家 {country_name} 搜索出 {len(sku_ls)} 商品,开始处理...")
if target_url is not None: # 从失败的链接继续
self.log(f"开始访问:{target_url}")
driver.tab.get(url=target_url)
driver.tab.wait.doc_loaded(timeout=60, raise_err=False)
result = []
# 处理所有需要审批的商品通过yield获取结果
for asin, status in driver.run_page_action():
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求停止处理ASIN", "WARNING")
break
self.log(f"ASIN {asin} 处理结果: {status}")
# 更新任务状态
if task_id in runing_task:
runing_task[task_id]["current_asin"] = asin
runing_task[task_id]["processed_asins"] += 1
runing_task[task_id]["failed_count"] += 1
result.append({
"asin": asin,
"status": status,
"done": False
})
if len(result) > 20:
self.post_result_batch(task_id, shop_name, country_code,result)
result = []
if len(result) > 0:
self.post_result_batch(task_id, shop_name, country_code, result)
self.log(f"国家 {country_name} 处理完成")
def post_stage_finished(self, task_id: int, user_id, stage_index):
if user_id in (None, "", 0):
raise ValueError("user_id is required for stage completion callback")
if stage_index is None:
raise ValueError("stage_index is required for stage completion callback")
url = f"{DELETE_BRAND_API_BASE}/api/shop-match/tasks/{task_id}/stage-finished"
payload = {"stage_index": stage_index}
params = {"user_id": user_id}
max_retries = 3
for retry in range(max_retries):
try:
self.log(f"Attempting stage completion callback ({retry + 1}/{max_retries})")
response = requests.post(
url,
params=params,
json=payload,
headers={"Content-Type": "application/json"},
timeout=30,
verify=False,
)
self.log(f"Stage completion callback response: {response.text}")
data = response.json() if response.text else {}
if response.status_code == 200 and isinstance(data, dict) and data.get("success"):
self.log(f"Stage completion callback succeeded: task={task_id}, stage={stage_index}")
return
self.log(f"Stage completion callback failed, status={response.status_code}", "WARNING")
except Exception as e:
self.log(f"Stage completion callback exception: {str(e)}", "ERROR")
if retry < max_retries - 1:
time.sleep(2)
raise RuntimeError(f"Stage completion callback failed after retries: task={task_id}, stage={stage_index}")
def post_result(self, task_id: int, shop_name: str, country_code: str, asin: str, status: str,is_done: bool = False):
"""回传处理结果到API
Args:
task_id: 任务ID
shop_name: 店铺名称
country_code: 国家代码
asin: ASIN
status: 处理状态
"""
url = f"{DELETE_BRAND_API_BASE}/api/shop-match/tasks/{task_id}/result"
payload = {
"shops": [
{
"error": "",
"countries": {
country_code: [
{
"asin": asin,
"status": status,
"done": is_done
}
]
},
"shopName": shop_name
}
]
}
max_retries = 3
for retry in range(max_retries):
try:
print("================【匹配】=====================")
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
self.log(f"回传URL: {url}")
self.log(f"回传数据: {payload}")
response = requests.post(
url,
json=payload,
headers={"Content-Type": "application/json"},
timeout=30,
verify=False
)
self.log(f"回传结果: {response.text}")
data = response.json() if response.text else {}
if response.status_code == 200 and isinstance(data, dict) and data.get("success"):
self.log(f"结果回传成功: {asin} - {status}")
return
else:
self.log(f"结果回传失败,状态码: {response.status_code}", "WARNING")
print("=====================================")
except Exception as e:
self.log(f"调用API异常: {str(e)}", "ERROR")
print("=====================================")
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(2)
self.log(f"已达到最大重试次数,结果回传最终失败", "ERROR")
raise RuntimeError("已达到最大重试次数,结果回传最终失败")
def post_result_batch(self, task_id: int, shop_name: str, country_code: str, country_data:list,
is_done: bool = False):
"""回传处理结果到API
Args:
task_id: 任务ID
shop_name: 店铺名称
country_code: 国家代码
asin: ASIN
status: 处理状态
"""
url = f"{DELETE_BRAND_API_BASE}/api/shop-match/tasks/{task_id}/result"
payload = {
"shops": [
{
"error": "",
"countries": {
country_code: country_data
},
"shopName": shop_name
}
]
}
max_retries = 3
for retry in range(max_retries):
try:
print("================【匹配】=====================")
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
self.log(f"回传URL: {url}")
self.log(f"回传数据: {payload}")
response = requests.post(
url,
json=payload,
headers={"Content-Type": "application/json"},
timeout=30,
verify=False
)
self.log(f"回传结果: {response.text}")
data = response.json() if response.text else {}
if response.status_code == 200 and isinstance(data, dict) and data.get("success"):
self.log(f"结果回传成功")
return
else:
self.log(f"结果回传失败,状态码: {response.status_code}", "WARNING")
print("=====================================")
except Exception as e:
self.log(f"调用API异常: {str(e)}", "ERROR")
print("=====================================")
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(2)
self.log(f"已达到最大重试次数,结果回传最终失败", "ERROR")
raise RuntimeError("已达到最大重试次数,结果回传最终失败")
if __name__ == "__main__":
user_info = {
"company": "rongchuang123",
"username": "自动化_Robot",
"password": "#20zsg25"
}
shop_name = "魏振峰"
country = "德国"
kill_process('v6')
driver = AmzoneMatchAction(user_info)
browser = driver.open_shop(shop_name)
sw_suc = driver.SwitchingCountries(country)
driver.SwitchPage()
risk_listing_filter = "All"
for _ in range(3):
try:
sku_ls = driver.search(filter_type=risk_listing_filter)
break
except Exception as e:
print(e)
driver.tab.refresh()
if len(sku_ls) > 0:
print("有数据,开始操作")
for asin, status in driver.run_page_action():
print(f"ASIN {asin} 的处理结果: {status}")
print("已完成操作")

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load Diff

View File

@@ -1,128 +0,0 @@
function textOf(node) {
return ((node && (node.innerText || node.textContent || '')) || '').replace(/\s+/g, ' ').trim();
}
function isVisible(el) {
if (!el || !el.getBoundingClientRect) return false;
const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden';
}
function clickElement(el) {
el.scrollIntoView({ block: 'center', inline: 'nearest' });
if (el.focus) el.focus();
for (const eventName of ['pointerdown', 'mousedown', 'pointerup', 'mouseup']) {
el.dispatchEvent(new MouseEvent(eventName, { bubbles: true, cancelable: true, view: window }));
}
el.click();
}
function summarize(el) {
return {
tag: el.tagName,
text: textOf(el).slice(0, 500),
attrs: Array.from(el.attributes || []).reduce((acc, attr) => {
acc[attr.name] = attr.value;
return acc;
}, {}),
outerHTML: (el.outerHTML || '').slice(0, 1000),
};
}
const view = document.querySelector('[data-testid="complete-drafts-view"]');
if (!view) return { ok: false, reason: 'complete drafts view not found' };
const selectedRows = Array.from(view.querySelectorAll('.ag-center-cols-container [role="row"]'))
.filter((row) => isVisible(row) && (row.getAttribute('aria-selected') === 'true' || row.classList.contains('ag-row-selected')));
if (!selectedRows.length) return { ok: false, reason: 'no selected complete draft rows' };
const bulkDropdown = view.querySelector('#bulk-dropdown');
if (!bulkDropdown) {
return {
ok: false,
reason: 'bulk dropdown not found',
selectedCount: selectedRows.length,
};
}
const toggle = bulkDropdown.shadowRoot && bulkDropdown.shadowRoot.querySelector('button[part="dropdown-button-toggle-button"], button');
clickElement(toggle || bulkDropdown);
function findBulkDeleteAction() {
const all = [];
const seen = new Set();
function walk(root) {
if (!root || seen.has(root)) return;
seen.add(root);
const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
let node = walker.currentNode;
while (node) {
if (node.nodeType === Node.ELEMENT_NODE && node.tagName) all.push(node);
if (node.shadowRoot) walk(node.shadowRoot);
if ((node.tagName || '').toLowerCase() === 'slot' && node.assignedElements) {
for (const assigned of node.assignedElements({ flatten: true })) walk(assigned);
}
node = walker.nextNode();
}
}
walk(document);
const exactAction = all.find((el) => {
const action = String(el.getAttribute && el.getAttribute('data-action') || '');
const text = textOf(el);
return isVisible(el) && action === 'DELETE_DRAFTS' && /删除商品信息草稿/.test(text);
});
if (exactAction) return exactAction;
return all.find((el) => {
const text = textOf(el);
const attrs = Array.from(el.attributes || []).map((attr) => `${attr.name}=${attr.value}`).join(' ');
const haystack = `${text} ${attrs}`;
if (!isVisible(el) || !/删除商品信息草稿/.test(haystack)) return false;
if (/^(HTML|BODY|KAT-DATA-GRID)$/i.test(el.tagName || '') && text.length > 80) return false;
if (/^DIV$/i.test(el.tagName || '') && text !== '删除商品信息草稿') return false;
if ((el.tagName || '').toLowerCase() === 'kat-icon') return false;
return /^(BUTTON|KAT-BUTTON|KAT-OPTION|KAT-DROPDOWN-OPTION|LI|A|SPAN|DIV)$/i.test(el.tagName || '');
});
}
const deleteAction = findBulkDeleteAction();
if (!deleteAction) {
const candidates = [];
const seen = new Set();
function walkCandidates(root) {
if (!root || seen.has(root)) return;
seen.add(root);
const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
let node = walker.currentNode;
while (node) {
if (node.nodeType === Node.ELEMENT_NODE && node.tagName && isVisible(node)) {
const text = textOf(node);
const attrs = Array.from(node.attributes || []).map((attr) => `${attr.name}=${attr.value}`).join(' ');
if (/删除|草稿|选择组操作|menu|option/i.test(`${text} ${attrs}`)) candidates.push(summarize(node));
}
if (node.shadowRoot) walkCandidates(node.shadowRoot);
node = walker.nextNode();
}
}
walkCandidates(document);
return {
ok: false,
reason: 'bulk delete draft action not found',
menuOpened: true,
selectedCount: selectedRows.length,
bulkDropdown: summarize(bulkDropdown),
candidates: candidates.slice(0, 50),
};
}
clickElement(deleteAction);
return {
ok: true,
selectedCount: selectedRows.length,
bulkDropdown: summarize(bulkDropdown),
clicked: summarize(deleteAction),
rows: selectedRows.slice(0, 10).map(summarize),
};

View File

@@ -1,55 +0,0 @@
function textOf(node) {
return (node && (node.innerText || node.textContent || '') || '').replace(/\s+/g, ' ').trim();
}
function attrsOf(el) {
const attrs = {};
if (!el || !el.attributes) return attrs;
for (const attr of el.attributes) attrs[attr.name] = attr.value;
return attrs;
}
function isVisible(el) {
if (!el || !el.getBoundingClientRect) return false;
const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden';
}
function walkRoots(root, rows, seen) {
if (!root || seen.has(root)) return;
seen.add(root);
const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
let node = walker.currentNode;
while (node) {
if (node.nodeType === Node.ELEMENT_NODE && node.tagName) rows.push(node);
if (node.shadowRoot) walkRoots(node.shadowRoot, rows, seen);
node = walker.nextNode();
}
}
function allElements() {
const rows = [];
walkRoots(document, rows, new Set());
return rows;
}
function rowOf(el, index) {
const attrs = attrsOf(el);
return {
index,
tag: el.tagName.toLowerCase(),
visible: isVisible(el),
text: textOf(el),
attrs,
outerHTML: (el.outerHTML || '').slice(0, 2500),
};
}
const candidates = allElements()
.map((el, index) => rowOf(el, index))
.filter((row) => row.visible && /补全草稿|快速查看|quick view|draft/i.test(`${row.tag} ${row.text} ${Object.values(row.attrs).join(' ')}`))
.slice(0, 120);
return candidates;

View File

@@ -1,73 +0,0 @@
function textOf(node) {
return (node && (node.innerText || node.textContent || '') || '').replace(/\s+/g, ' ').trim();
}
function attrsOf(el) {
const attrs = {};
if (!el || !el.attributes) return attrs;
for (const attr of el.attributes) attrs[attr.name] = attr.value;
return attrs;
}
function isVisible(el) {
if (!el || !el.getBoundingClientRect) return false;
const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden';
}
function walkRoots(root, rows, seen) {
if (!root || seen.has(root)) return;
seen.add(root);
const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
let node = walker.currentNode;
while (node) {
if (node.nodeType === Node.ELEMENT_NODE && node.tagName) rows.push(node);
if (node.shadowRoot) walkRoots(node.shadowRoot, rows, seen);
node = walker.nextNode();
}
}
function allElements() {
const rows = [];
walkRoots(document, rows, new Set());
return rows;
}
function attrsText(attrs) {
return Object.entries(attrs).map(([key, value]) => `${key}=${value}`).join(' ');
}
const pattern = /快速查看|quick|draft|草稿|提交|submitted|missing|inventory|myinventory/i;
const elements = allElements()
.map((el, index) => {
const attrs = attrsOf(el);
const text = textOf(el);
return {
index,
tag: el.tagName.toLowerCase(),
visible: isVisible(el),
text: text.slice(0, 500),
attrs,
outerHTML: (el.outerHTML || '').slice(0, 1000),
haystack: `${el.tagName} ${text} ${attrsText(attrs)}`,
};
})
.filter((row) => pattern.test(row.haystack))
.map(({ haystack, ...row }) => row)
.slice(0, 80);
const resources = performance
.getEntriesByType('resource')
.map((entry) => entry.name)
.filter((url) => pattern.test(url))
.slice(-120);
return {
location: window.location.href,
readyState: document.readyState,
capturedResponses: (window.__crawlerCompleteDraftResponses || []).slice(-10),
elements,
resources,
};

View File

@@ -1,121 +0,0 @@
function textOf(node) {
return (node && (node.innerText || node.textContent || '') || '').replace(/\s+/g, ' ').trim();
}
function attrsOf(el) {
const attrs = {};
if (!el || !el.attributes) return attrs;
for (const attr of el.attributes) attrs[attr.name] = attr.value;
return attrs;
}
function isVisible(el) {
if (!el || !el.getBoundingClientRect) return false;
const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden';
}
function walkRoots(root, rows, seen) {
if (!root || seen.has(root)) return;
seen.add(root);
const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
let node = walker.currentNode;
while (node) {
if (node.nodeType === Node.ELEMENT_NODE && node.tagName) rows.push(node);
if (node.shadowRoot) walkRoots(node.shadowRoot, rows, seen);
node = walker.nextNode();
}
}
function allElements() {
const rows = [];
walkRoots(document, rows, new Set());
return rows;
}
function hasCountText(text) {
return /[(]\s*[\d,]+\s*[)]/.test(text);
}
function insideListingRow(el) {
return Boolean(el.closest('[data-sku], tr, [role="row"], [data-testid*="row"]'));
}
function rowOf(el, index) {
const attrs = attrsOf(el);
const text = textOf(el);
const labelText = attrs.label || attrs.title || attrs['aria-label'] || '';
const valueText = attrs.value || attrs.count || attrs['data-count'] || attrs['data-value'] || '';
return {
index,
raw_text: text,
label_text: labelText,
quantity_text: valueText,
attrs,
visible: isVisible(el),
outerHTML: (el.outerHTML || '').slice(0, 1200),
};
}
function splitCountRow(el, index) {
const countText = textOf(el);
if (!/^\s*[\d,]+\s*$/.test(countText)) return null;
let parent = el.parentElement;
for (let depth = 0; parent && depth < 4; depth += 1, parent = parent.parentElement) {
const parentText = textOf(parent);
const labelText = parentText
.replace(new RegExp(`(^|\\s)${countText.replace(/,/g, '\\,')}(\\s|$)`), ' ')
.replace(/[(]\s*[\d,]+\s*[)]/g, ' ')
.replace(/\s+/g, ' ')
.trim();
if (labelText && labelText !== parentText && !/^\s*[\d,]+\s*$/.test(labelText)) {
return {
index,
raw_text: '',
label_text: labelText,
quantity_text: countText,
attrs: attrsOf(parent),
visible: isVisible(parent),
outerHTML: (parent.outerHTML || '').slice(0, 1200),
};
}
}
return null;
}
const regionRows = allElements().filter((el) => isVisible(el) && !insideListingRow(el));
const byText = new Map();
regionRows
.forEach((el, index) => {
const row = rowOf(el, index);
const combined = `${row.raw_text} ${row.label_text} ${row.quantity_text}`;
if (row.raw_text.length > 200) return;
if (/[(]\s*[\d,]+\s*[)]/.test(combined)) {
const key = combined;
if (!byText.has(key)) byText.set(key, row);
return;
}
const splitRow = splitCountRow(el, index);
if (splitRow) {
const key = `${splitRow.label_text} ${splitRow.quantity_text}`;
if (!byText.has(key)) byText.set(key, splitRow);
}
});
return {
region: {
url: window.location.href,
title: document.title,
},
rows: Array.from(byText.values()).filter((row) => {
const combined = `${row.raw_text} ${row.label_text}`.toLowerCase();
return !/所有商品|启售商品|改善商品信息/.test(combined);
}),
candidates: Array.from(byText.values()).slice(0, 80),
};

View File

@@ -1,100 +0,0 @@
function textOf(node) {
return ((node && (node.innerText || node.textContent || '')) || '').replace(/\s+/g, ' ').trim();
}
function isVisible(el) {
if (!el || !el.getBoundingClientRect) return false;
const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden';
}
function clickElement(el) {
el.scrollIntoView({ block: 'center', inline: 'nearest' });
if (el.focus) el.focus();
for (const eventName of ['pointerdown', 'mousedown', 'pointerup', 'mouseup']) {
el.dispatchEvent(new MouseEvent(eventName, { bubbles: true, cancelable: true, view: window }));
}
el.click();
}
function summarizeRow(row) {
return {
rowId: row.getAttribute('row-id') || '',
text: textOf(row).slice(0, 500),
selected: row.getAttribute('aria-selected') === 'true' || row.classList.contains('ag-row-selected'),
};
}
const payload = arguments[0] || {};
const view = document.querySelector('[data-testid="complete-drafts-view"]');
if (!view) return { ok: false, reason: 'complete drafts view not found' };
const rows = Array.from(view.querySelectorAll('.ag-center-cols-container [role="row"]'))
.filter((row) => isVisible(row));
const rowCount = rows.length;
const selectedBefore = rows.filter((row) => row.getAttribute('aria-selected') === 'true' || row.classList.contains('ag-row-selected')).length;
const maxSelectCountRaw = Number(payload.maxSelectCount || 0);
const maxSelectCount = Number.isFinite(maxSelectCountRaw) && maxSelectCountRaw > 0 ? Math.floor(maxSelectCountRaw) : 0;
if (payload.probeOnly) {
return {
ok: true,
probeOnly: true,
rowCount,
selectedCount: selectedBefore,
rows: rows.slice(0, 10).map(summarizeRow),
};
}
if (rowCount <= 0) {
return { ok: false, reason: 'no complete draft rows', rowCount, selectedCount: selectedBefore };
}
const headerCheckbox = view.querySelector('.ag-header-select-all input[type="checkbox"]');
if (!headerCheckbox) {
return { ok: false, reason: 'complete draft select-all checkbox not found', rowCount, selectedCount: selectedBefore };
}
if (maxSelectCount > 0 && maxSelectCount < rowCount) {
if (headerCheckbox.checked) {
clickElement(headerCheckbox);
}
for (const row of rows) {
if (row.getAttribute('aria-selected') === 'true' || row.classList.contains('ag-row-selected')) {
const checkbox = row.querySelector('input[type="checkbox"]');
if (checkbox && checkbox.checked) clickElement(checkbox);
}
}
for (const row of rows.slice(0, maxSelectCount)) {
const checkbox = row.querySelector('input[type="checkbox"]');
if (checkbox && !checkbox.checked) {
clickElement(checkbox);
} else if (!checkbox) {
clickElement(row);
}
}
} else if (!headerCheckbox.checked || selectedBefore < rowCount) {
clickElement(headerCheckbox);
}
let selectedRows = Array.from(view.querySelectorAll('.ag-center-cols-container [role="row"]'))
.filter((row) => isVisible(row) && (row.getAttribute('aria-selected') === 'true' || row.classList.contains('ag-row-selected')));
if (selectedRows.length <= 0) {
const firstRowCheckbox = rows[0] && rows[0].querySelector('input[type="checkbox"]');
if (firstRowCheckbox && !firstRowCheckbox.checked) {
clickElement(firstRowCheckbox);
}
}
selectedRows = Array.from(view.querySelectorAll('.ag-center-cols-container [role="row"]'))
.filter((row) => isVisible(row) && (row.getAttribute('aria-selected') === 'true' || row.classList.contains('ag-row-selected')));
return {
ok: selectedRows.length > 0,
rowCount,
selectedCount: selectedRows.length,
headerChecked: headerCheckbox.checked,
rows: selectedRows.slice(0, 10).map(summarizeRow),
};

View File

@@ -1,59 +0,0 @@
function textOf(node) {
return ((node && (node.innerText || node.textContent || '')) || '').replace(/\s+/g, ' ').trim();
}
function isVisible(el) {
if (!el || !el.getBoundingClientRect) return false;
const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden';
}
function clickElement(el) {
el.scrollIntoView({ block: 'center', inline: 'nearest' });
if (el.focus) el.focus();
for (const eventName of ['pointerdown', 'mousedown', 'pointerup', 'mouseup']) {
el.dispatchEvent(new MouseEvent(eventName, { bubbles: true, cancelable: true, view: window }));
}
el.click();
}
function summarize(el) {
return {
tag: el.tagName,
text: textOf(el),
className: el.className || '',
outerHTML: (el.outerHTML || '').slice(0, 1000),
};
}
const targetTag = String((arguments[0] && arguments[0].tag) || '').replace(/\s+/g, ' ').trim();
if (!targetTag) return { ok: false, reason: 'target tag empty' };
const view = document.querySelector('[data-testid="complete-drafts-view"]');
if (!view) return { ok: false, reason: 'complete drafts view not found' };
const chips = Array.from(view.querySelectorAll('.JanusChip-module__janusChipContainer--gRjdp'));
const chip = chips.find((el) => {
const text = textOf(el);
return text === targetTag || text.startsWith(`${targetTag} (`) || text.startsWith(`${targetTag}`);
});
if (!chip) {
return {
ok: false,
reason: 'complete draft chip not found',
tag: targetTag,
chips: chips.map(summarize),
};
}
if (!/active/.test(String(chip.className || ''))) {
clickElement(chip);
}
return {
ok: true,
tag: targetTag,
clicked: summarize(chip),
};

View File

@@ -1,89 +0,0 @@
function textOf(node) {
return ((node && (node.innerText || node.textContent || '')) || '').replace(/\s+/g, ' ').trim();
}
function attrsText(el) {
if (!el || !el.attributes) return '';
return Array.from(el.attributes).map((attr) => `${attr.name}=${attr.value}`).join(' ');
}
function isVisible(el) {
if (!el || !el.getBoundingClientRect) return false;
const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden';
}
function walkRoots(root, rows, seen) {
if (!root || seen.has(root)) return;
seen.add(root);
const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
let node = walker.currentNode;
while (node) {
if (node.nodeType === Node.ELEMENT_NODE && node.tagName) rows.push(node);
if (node.shadowRoot) walkRoots(node.shadowRoot, rows, seen);
if ((node.tagName || '').toLowerCase() === 'slot' && node.assignedElements) {
for (const assigned of node.assignedElements({ flatten: true })) {
walkRoots(assigned, rows, seen);
}
}
node = walker.nextNode();
}
}
function allElements(root) {
const rows = [];
walkRoots(root || document, rows, new Set());
return rows;
}
function clickElement(el) {
el.scrollIntoView({ block: 'center', inline: 'nearest' });
if (el.focus) el.focus();
for (const eventName of ['pointerdown', 'mousedown', 'pointerup', 'mouseup']) {
el.dispatchEvent(new MouseEvent(eventName, { bubbles: true, cancelable: true, view: window }));
}
el.click();
}
function clickTargetFor(el) {
if (el && el.shadowRoot) {
const button = el.shadowRoot.querySelector('button:not([disabled]), [role="button"]:not([disabled])');
if (button) return button;
}
return el;
}
const modal = document.querySelector('kat-modal[data-testid="action-modal"]') ||
allElements(document).find((el) => /删除|Delete/i.test(textOf(el)) && (el.tagName || '').toLowerCase() === 'kat-modal');
if (!modal) return { ok: false, reason: 'delete modal not found' };
const buttons = allElements(modal).filter((el) => {
const haystack = `${el.tagName || ''} ${attrsText(el)} ${textOf(el)}`;
return isVisible(el) && /kat-button|button/i.test(el.tagName || '') && /variant=primary|删除|确认|Delete|Confirm/i.test(haystack);
});
const button = buttons[0];
if (!button) {
return {
ok: false,
reason: 'confirm button not found',
modalText: textOf(modal),
modalHTML: (modal.outerHTML || '').slice(0, 2000),
};
}
const target = clickTargetFor(button);
clickElement(target);
return {
ok: true,
clicked: {
tag: button.tagName,
text: textOf(button),
attrs: attrsText(button),
targetTag: target.tagName,
},
};

View File

@@ -1,46 +0,0 @@
function textOf(node) {
return ((node && (node.innerText || node.textContent || '')) || '').replace(/\s+/g, ' ').trim();
}
function attrsText(el) {
if (!el || !el.attributes) return '';
return Array.from(el.attributes).map((attr) => `${attr.name}=${attr.value}`).join(' ');
}
function isVisible(el) {
if (!el || !el.getBoundingClientRect) return false;
const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden';
}
function walkRoots(root, rows, seen) {
if (!root || seen.has(root)) return;
seen.add(root);
const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
let node = walker.currentNode;
while (node) {
if (node.nodeType === Node.ELEMENT_NODE && node.tagName) rows.push(node);
if (node.shadowRoot) walkRoots(node.shadowRoot, rows, seen);
if ((node.tagName || '').toLowerCase() === 'slot' && node.assignedElements) {
for (const assigned of node.assignedElements({ flatten: true })) {
walkRoots(assigned, rows, seen);
}
}
node = walker.nextNode();
}
}
const rows = [];
walkRoots(document, rows, new Set());
const seen = new Set();
return rows
.filter((el) => {
const haystack = `${el.tagName || ''} ${attrsText(el)} ${textOf(el)}`;
return isVisible(el) && /kat-alert|alert|toast|notification|已删除|已经删除|15\s*分钟|success/i.test(haystack);
})
.map((el) => textOf(el) || attrsText(el))
.filter((text) => text && !seen.has(text) && seen.add(text))
.slice(0, 30);

View File

@@ -1,187 +0,0 @@
function textOf(node) {
return ((node && (node.innerText || node.textContent || '')) || '').replace(/\s+/g, ' ').trim();
}
function attrsText(el) {
if (!el || !el.attributes) return '';
return Array.from(el.attributes).map((attr) => `${attr.name}=${attr.value}`).join(' ');
}
function isVisible(el) {
if (!el || !el.getBoundingClientRect) return false;
const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden';
}
function walkRoots(root, rows, seen) {
if (!root || seen.has(root)) return;
seen.add(root);
const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
let node = walker.currentNode;
while (node) {
if (node.nodeType === Node.ELEMENT_NODE && node.tagName) rows.push(node);
if (node.shadowRoot) walkRoots(node.shadowRoot, rows, seen);
if ((node.tagName || '').toLowerCase() === 'slot' && node.assignedElements) {
for (const assigned of node.assignedElements({ flatten: true })) {
walkRoots(assigned, rows, seen);
}
}
node = walker.nextNode();
}
}
function allElements(root) {
const rows = [];
walkRoots(root || document, rows, new Set());
return rows;
}
function closestDeep(el, selector) {
let node = el;
while (node) {
if (node.closest) {
const found = node.closest(selector);
if (found) return found;
}
const root = node.getRootNode && node.getRootNode();
node = root && root.host ? root.host : null;
}
return null;
}
function insideInventoryRow(el) {
return Boolean(closestDeep(el, 'div[data-sku]'));
}
function isDeleteAction(el) {
const haystack = `${el.tagName || ''} ${attrsText(el)} ${textOf(el)}`;
return /data-action=DeleteListing|action=DeleteListing|删除商品信息|删除商品和报价|DeleteListing|delete\s+listing/i.test(haystack);
}
function clickElement(el) {
el.scrollIntoView({ block: 'center', inline: 'nearest' });
if (el.focus) el.focus();
for (const eventName of ['pointerdown', 'mousedown', 'pointerup', 'mouseup']) {
el.dispatchEvent(new MouseEvent(eventName, { bubbles: true, cancelable: true, view: window }));
}
el.click();
}
function hostDropdownOf(el) {
const host = closestDeep(el, 'kat-dropdown-button');
return host || ((el.tagName || '').toLowerCase() === 'kat-dropdown-button' ? el : null);
}
function dropdownToggleOf(dropdown, fallback) {
if (!dropdown) return fallback;
if (fallback && textOf(fallback)) return fallback;
return fallback || dropdown;
}
function openBulkActionMenu(candidate) {
const dropdown = hostDropdownOf(candidate);
const toggle = dropdownToggleOf(dropdown, candidate);
clickElement(toggle);
return { dropdown, toggle };
}
function summarize(el) {
return {
tag: el.tagName,
text: textOf(el),
attrs: attrsText(el),
visible: isVisible(el),
outerHTML: (el.outerHTML || '').slice(0, 1000),
};
}
function findVisibleBulkDeleteAction() {
return allElements(document).find((el) => isDeleteAction(el) && isVisible(el) && !insideInventoryRow(el));
}
function findDeleteActionAfterMenuOpen(candidate) {
const dropdown = hostDropdownOf(candidate) || candidate;
const exactMenuButton = allElements(dropdown).find((child) => {
const tag = (child.tagName || '').toLowerCase();
const role = child.getAttribute && child.getAttribute('role');
return tag === 'button' && role === 'menuitem' && isDeleteAction(child);
});
if (exactMenuButton) return { mode: 'bulk-action-menuitem-delete', el: exactMenuButton };
const nestedDeleteAction = allElements(dropdown).find((child) => isDeleteAction(child) && isVisible(child));
if (nestedDeleteAction) return { mode: 'bulk-action-nested-delete', el: nestedDeleteAction };
const deleteAction = findVisibleBulkDeleteAction();
if (deleteAction) return { mode: 'bulk-action-visible-delete', el: deleteAction };
return null;
}
function isBulkActionCandidate(el) {
const tag = (el.tagName || '').toLowerCase();
if (!/^(button|kat-button|kat-dropdown-button|kat-select|kat-popover|kat-menu-button)$/.test(tag)) return false;
if (!isVisible(el) || insideInventoryRow(el)) return false;
const haystack = `${tag} ${attrsText(el)} ${textOf(el)}`;
return /DeleteListing|删除商品信息|删除|操作|更多|Action|Actions|selected|选定|已选择|批量/i.test(haystack);
}
const directDelete = findVisibleBulkDeleteAction();
if (directDelete) {
clickElement(directDelete);
return { ok: true, mode: 'direct-delete-button', clicked: summarize(directDelete) };
}
const rows = allElements(document);
const candidates = rows.filter(isBulkActionCandidate);
for (const candidate of candidates) {
try {
const openResult = openBulkActionMenu(candidate);
const deleteAction = findDeleteActionAfterMenuOpen(candidate);
if (deleteAction) {
clickElement(deleteAction.el);
return {
ok: true,
mode: deleteAction.mode,
candidate: summarize(candidate),
opened: {
dropdown: openResult.dropdown ? summarize(openResult.dropdown) : null,
toggle: openResult.toggle ? summarize(openResult.toggle) : null,
},
clicked: summarize(deleteAction.el),
};
}
return {
ok: false,
reason: 'bulk action menu opened; delete action not visible yet',
menuOpened: true,
candidate: summarize(candidate),
opened: {
dropdown: openResult.dropdown ? summarize(openResult.dropdown) : null,
toggle: openResult.toggle ? summarize(openResult.toggle) : null,
},
deleteLikeElements: allElements(document)
.filter((el) => /DeleteListing|删除商品信息|删除商品和报价|delete\s+listing/i.test(`${el.tagName || ''} ${attrsText(el)} ${textOf(el)}`) && !insideInventoryRow(el))
.slice(0, 20)
.map(summarize),
};
} catch (error) {
// Try the next non-row bulk action control.
}
}
return {
ok: false,
reason: 'bulk delete action not found',
candidates: candidates.slice(0, 20).map(summarize),
deleteLikeElements: rows
.filter((el) => /DeleteListing|删除商品信息|删除商品和报价|delete\s+listing/i.test(`${el.tagName || ''} ${attrsText(el)} ${textOf(el)}`) && !insideInventoryRow(el))
.slice(0, 20)
.map(summarize),
visibleButtons: rows
.filter((el) => /^(button|kat-button|kat-dropdown-button)$/i.test(el.tagName || '') && isVisible(el))
.slice(0, 20)
.map(summarize),
};

View File

@@ -1,71 +0,0 @@
function isVisible(el) {
if (!el || !el.getBoundingClientRect) return false;
const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden';
}
function walkRoots(root, rows, seen) {
if (!root || seen.has(root)) return;
seen.add(root);
const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
let node = walker.currentNode;
while (node) {
if (node.nodeType === Node.ELEMENT_NODE && node.tagName) rows.push(node);
if (node.shadowRoot) walkRoots(node.shadowRoot, rows, seen);
node = walker.nextNode();
}
}
function allElements() {
const rows = [];
walkRoots(document, rows, new Set());
return rows;
}
function insideInventoryRow(el) {
return Boolean(el && el.closest && el.closest('div[data-sku]'));
}
function clickTargetFor(checkbox) {
if (!checkbox) return null;
if (checkbox.shadowRoot) {
const shadowTarget = checkbox.shadowRoot.querySelector('[role="checkbox"], .checkbox');
if (shadowTarget) return shadowTarget;
}
return checkbox;
}
const rowCount = document.querySelectorAll('div[data-sku]').length;
if (rowCount <= 0) {
return { ok: false, reason: 'no inventory rows after status filter', rowCount };
}
const checkboxes = allElements()
.filter((el) => (el.tagName || '').toLowerCase() === 'kat-checkbox' && isVisible(el))
.sort((a, b) => {
const rectA = a.getBoundingClientRect();
const rectB = b.getBoundingClientRect();
return rectA.top - rectB.top || rectA.left - rectB.left;
});
const headerCheckbox = checkboxes.find((el) => !insideInventoryRow(el)) || checkboxes[0];
if (!headerCheckbox) {
return { ok: false, reason: 'select-all checkbox not found', rowCount, checkboxCount: checkboxes.length };
}
const target = clickTargetFor(headerCheckbox);
target.scrollIntoView({ block: 'center', inline: 'nearest' });
target.click();
return {
ok: true,
rowCount,
checkboxCount: checkboxes.length,
clicked: {
tag: headerCheckbox.tagName,
insideInventoryRow: insideInventoryRow(headerCheckbox),
outerHTML: (headerCheckbox.outerHTML || '').slice(0, 1000),
},
};

View File

@@ -1,124 +0,0 @@
function textOf(node) {
return ((node && (node.innerText || node.textContent || '')) || '').replace(/\s+/g, ' ').trim();
}
function normalize(text) {
return String(text || '').replace(/\s+/g, ' ').trim();
}
function isVisible(el) {
if (!el || !el.getBoundingClientRect) return false;
const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden';
}
function walkRoots(root, rows, seen) {
if (!root || seen.has(root)) return;
seen.add(root);
const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
let node = walker.currentNode;
while (node) {
if (node.nodeType === Node.ELEMENT_NODE && node.tagName) rows.push(node);
if (node.shadowRoot) walkRoots(node.shadowRoot, rows, seen);
node = walker.nextNode();
}
}
function allElements() {
const rows = [];
walkRoots(document, rows, new Set());
return rows;
}
function parseStatus(rawText) {
const text = normalize(rawText);
if (!text) return '';
const match = text.match(/^(.*?)\s*[(]\s*[\d,]+\s*[)]\s*$/);
return normalize(match ? match[1] : text);
}
function collectStatusTargets() {
const pattern = /(全部|在搜索结果中禁止显示|配送问题|缺少报价|已停售|详情页面已删除|订单页面已删除|商品信息草稿|缺少的信息|定价问题|需要批准|不可售|在售)\s*[(]?\s*[\d,]*\s*[)]?/g;
const hits = [];
for (const el of allElements()) {
if (!isVisible(el)) continue;
const text = normalize(textOf(el));
if (!text) continue;
const match = text.match(pattern);
if (!match) continue;
const parsed = parseStatus(match[0]);
hits.push({
el,
text,
parsed,
tag: el.tagName,
clickable: Boolean(el.onclick) || el.matches('button,[role="button"],kat-option,kat-label,a,[tabindex]'),
});
}
return hits;
}
const targetStatus = normalize((__crawlerPayload || {}).status);
const candidates = collectStatusTargets()
.filter((item) => item.parsed === targetStatus || item.text.includes(targetStatus))
.map((item) => ({
...item,
score:
(item.parsed === targetStatus ? 300 : 0) +
(item.clickable ? 120 : 0) +
(/^[^ ]+\s*[(]\s*[\d,]+\s*[)]$/.test(item.text) ? 80 : 0) +
(/button|option|label/i.test(item.tag) ? 30 : 0),
}))
.sort((a, b) => b.score - a.score);
if (!candidates.length) {
return { ok: false, reason: 'visible status target not found', targetStatus };
}
for (const candidate of candidates) {
const chain = [candidate.el];
let node = candidate.el.parentElement;
for (let depth = 0; node && depth < 5; depth += 1, node = node.parentElement) {
chain.push(node);
}
for (const el of chain) {
try {
el.scrollIntoView({ block: 'center', inline: 'nearest' });
el.click();
return {
ok: true,
targetStatus,
clicked: {
tag: el.tagName,
text: normalize(textOf(el)).slice(0, 300),
},
matched: {
tag: candidate.tag,
text: candidate.text.slice(0, 300),
parsed: candidate.parsed,
score: candidate.score,
},
};
} catch (error) {
// try parent
}
}
}
return {
ok: false,
reason: 'visible status target click failed',
targetStatus,
candidates: candidates.slice(0, 10).map((item) => ({
tag: item.tag,
text: item.text.slice(0, 300),
parsed: item.parsed,
score: item.score,
clickable: item.clickable,
})),
};

View File

@@ -1,3 +0,0 @@
document.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', code: 'Escape', bubbles: true }));
document.body.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
return true;

View File

@@ -1,95 +0,0 @@
const keywords = ['ListingStatus', 'listing status', '商品状态', '商品狀態', 'status', 'filter', '库存'];
function textOf(node) {
return (node && (node.innerText || node.textContent || '') || '').replace(/\s+/g, ' ').trim();
}
function attrsOf(el) {
const attrs = {};
if (!el || !el.attributes) return attrs;
for (const attr of el.attributes) attrs[attr.name] = attr.value;
return attrs;
}
function isVisible(el) {
if (!el || !el.getBoundingClientRect) return false;
const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden';
}
function walkRoots(root, rows, seen) {
if (!root || seen.has(root)) return;
seen.add(root);
const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
let node = walker.currentNode;
while (node) {
if (node.nodeType === Node.ELEMENT_NODE && node.tagName) rows.push(node);
if (node.shadowRoot) walkRoots(node.shadowRoot, rows, seen);
node = walker.nextNode();
}
}
function allElements() {
const rows = [];
walkRoots(document, rows, new Set());
return rows;
}
function nearestLabel(el) {
const parts = [];
let node = el;
for (let depth = 0; node && depth < 6; depth += 1, node = node.parentElement) {
const text = textOf(node);
if (text) parts.push(text.slice(0, 400));
}
const id = el.getAttribute('id');
if (id) {
document.querySelectorAll(`label[for="${CSS.escape(id)}"]`).forEach((label) => {
const text = textOf(label);
if (text) parts.unshift(text);
});
}
return [...new Set(parts)].join(' | ');
}
function hasKeyword(text) {
return keywords.some((keyword) => text.toLowerCase().includes(keyword.toLowerCase()));
}
function candidateRow(el, index) {
const attrs = attrsOf(el);
const attrText = Object.entries(attrs).map(([k, v]) => `${k}=${v}`).join(' ');
const label = nearestLabel(el);
const text = textOf(el);
const haystack = `${el.tagName} ${attrText} ${text} ${label}`;
let score = 0;
if (/商品状态|商品狀態|ListingStatus|listing status/i.test(haystack)) score += 100;
if (/status/i.test(haystack)) score += 25;
if (/filter|库存/i.test(haystack)) score += 10;
if (isVisible(el)) score += 5;
if (el.closest('[data-sku], tr, [role="row"]')) score -= 50;
return {
index,
score,
tag: el.tagName.toLowerCase(),
visible: isVisible(el),
text,
label,
attrs,
outerHTML: (el.outerHTML || '').slice(0, 2000),
};
}
const rows = allElements();
const candidates = [];
rows.forEach((el, index) => {
const tag = (el.tagName || '').toLowerCase();
const haystack = `${tag} ${textOf(el)} ${Object.values(attrsOf(el)).join(' ')} ${nearestLabel(el)}`;
if (tag === 'kat-dropdown' || tag === 'kat-option' || hasKeyword(haystack)) {
candidates.push(candidateRow(el, index));
}
});
return candidates.sort((a, b) => b.score - a.score).slice(0, 80);

View File

@@ -1,295 +0,0 @@
function textOf(node) {
return (node && (node.innerText || node.textContent || '') || '').replace(/\s+/g, ' ').trim();
}
function normalize(text) {
return String(text || '').replace(/\s+/g, ' ').trim();
}
function attrsText(el) {
if (!el || !el.attributes) return '';
return Array.from(el.attributes).map((attr) => `${attr.name}=${attr.value}`).join(' ');
}
function attrsOf(el) {
const attrs = {};
if (!el || !el.attributes) return attrs;
for (const attr of el.attributes) attrs[attr.name] = attr.value;
return attrs;
}
function isDisabled(el) {
if (!el) return false;
return (
el.hasAttribute('disabled') ||
el.disabled === true ||
el.getAttribute('aria-disabled') === 'true' ||
/\bdisabled\b/i.test(el.getAttribute('class') || '')
);
}
function isVisible(el) {
if (!el || !el.getBoundingClientRect) return false;
const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden';
}
function walkRoots(root, rows, seen) {
if (!root || seen.has(root)) return;
seen.add(root);
const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
let node = walker.currentNode;
while (node) {
if (node.nodeType === Node.ELEMENT_NODE && node.tagName) rows.push(node);
if (node.shadowRoot) walkRoots(node.shadowRoot, rows, seen);
node = walker.nextNode();
}
}
function allElements() {
const rows = [];
walkRoots(document, rows, new Set());
return rows;
}
function nearestLabel(el) {
const parts = [];
let node = el;
for (let depth = 0; node && depth < 7; depth += 1, node = node.parentElement) {
const text = textOf(node);
if (text) parts.push(text.slice(0, 500));
}
const id = el.getAttribute('id');
if (id) {
document.querySelectorAll(`label[for="${CSS.escape(id)}"]`).forEach((label) => {
const text = textOf(label);
if (text) parts.unshift(text);
});
}
return [...new Set(parts)].join(' | ');
}
function optionTextOf(dropdown) {
const roots = [dropdown];
if (dropdown.shadowRoot) roots.push(dropdown.shadowRoot);
const texts = [];
for (const root of roots) {
for (const option of Array.from(root.querySelectorAll('kat-option, kat-label, [role="option"], [class*="parentKatOptionStyle"]'))) {
texts.push(`${textOf(option)} ${attrsText(option)}`);
}
}
return normalize(texts.join(' '));
}
function isSearchFieldDropdown(text) {
return /(^|\s)(SKUS?|ASIN|FNSKU|UPC\/EAN|TITLE_KEYWORD|GTIN)(\s|$)|商品名称\/关键字|data-value=(SKU|ASIN|FNSKU|TITLE_KEYWORD|GTIN)|value=(SKU|ASIN|FNSKU|TITLE_KEYWORD|GTIN)/i.test(text);
}
function statusHitCount(text) {
const matches = normalize(text).match(
/(全部|在售|不可售|详情页面已删除|订单页面已删除|商品信息草稿|缺少的信息|定价问题|需要批准|配送问题|缺少报价|已停售|在搜索结果中禁止显示)\s*(?:[(]\s*[\d,]+\s*[)])?/g
);
return matches ? matches.length : 0;
}
function scoreDropdown(el) {
const text = textOf(el);
const label = nearestLabel(el);
const attrs = attrsText(el);
const optionText = optionTextOf(el);
const haystack = `${el.tagName} ${attrs} ${text} ${label} ${optionText}`;
const statusHits = statusHitCount(`${text} ${label} ${optionText}`);
const isStatusDropdown = /statusDropdown|商品状态|商品狀態|ListingStatus|listing status/i.test(haystack);
let score = 0;
if (isSearchFieldDropdown(haystack)) score -= 1000;
if (isStatusDropdown) score += 500;
if (/商品状态|商品狀態|ListingStatus|listing status/i.test(haystack)) score += 120;
if (/\bstatus\b|状态/i.test(haystack)) score += 25;
if (/filter|筛选|筛選|库存/i.test(haystack)) score += 10;
if (/全部\s*[(]\s*[\d,]+\s*[)]/.test(`${text} ${label} ${optionText}`)) score += 300;
if (statusHits >= 2) score += statusHits * 180;
if (!isStatusDropdown && statusHits === 0) score -= 200;
if (isVisible(el)) score += 10;
if (el.closest('[data-sku], tr, [role="row"], [data-testid*="row"]')) score -= 80;
if (isDisabled(el)) score -= 20;
return { score, text, label, haystack, statusHits, disabled: isDisabled(el) };
}
function candidateTriggers(dropdown) {
const selectors = [
'kat-dropdown-button',
'kat-button',
'button',
'[role="button"]',
'[aria-haspopup]',
'[part="trigger"]',
'[part="button"]',
'.header',
'.trigger',
'.button',
'span',
'div',
];
const seen = new Set();
const triggers = [];
const roots = [dropdown];
if (dropdown.shadowRoot) roots.push(dropdown.shadowRoot);
for (const root of roots) {
for (const selector of selectors) {
for (const el of Array.from(root.querySelectorAll(selector))) {
if (!el || seen.has(el)) continue;
seen.add(el);
const text = textOf(el);
const attrs = attrsText(el);
const score =
(isVisible(el) ? 20 : 0) +
(/全部\s*[(]\s*[\d,]+\s*[)]/.test(text) ? 200 : 0) +
(/商品状态|商品狀態|ListingStatus|listing status/i.test(`${text} ${attrs}`) ? 120 : 0) +
(/button|trigger|toggle|dropdown|expand/i.test(`${el.tagName} ${attrs}`) ? 40 : 0);
triggers.push({ el, text, attrs, score });
}
}
}
return triggers
.filter((item) => item.score > 0)
.sort((a, b) => b.score - a.score);
}
function clickTrigger(dropdown) {
const triggers = candidateTriggers(dropdown);
for (const trigger of triggers) {
try {
trigger.el.scrollIntoView({ block: 'center', inline: 'nearest' });
trigger.el.click();
return {
ok: true,
trigger: {
tag: trigger.el.tagName,
text: trigger.text,
attrs: trigger.attrs,
score: trigger.score,
},
};
} catch (error) {
// continue
}
}
try {
dropdown.scrollIntoView({ block: 'center', inline: 'nearest' });
dropdown.click();
return {
ok: true,
trigger: {
tag: dropdown.tagName,
text: textOf(dropdown),
attrs: attrsText(dropdown),
score: -1,
},
};
} catch (error) {
return {
ok: false,
reason: String(error),
tried: triggers.slice(0, 10).map((item) => ({
tag: item.el.tagName,
text: item.text,
attrs: item.attrs,
score: item.score,
})),
};
}
}
const dropdowns = allElements().filter((el) => el.tagName.toLowerCase() === 'kat-dropdown');
const ranked = dropdowns
.map((el, index) => ({ el, index, ...scoreDropdown(el) }))
.filter((row) => row.score > 0)
.sort((a, b) => b.score - a.score);
if (!ranked.length) {
return { ok: false, reason: 'listing status kat-dropdown not found', candidates: [] };
}
const selected = ranked[0];
if (selected.disabled) {
return {
ok: false,
reason: 'listing status dropdown disabled/loading',
selected: {
index: selected.index,
score: selected.score,
statusHits: selected.statusHits,
text: selected.text,
label: selected.label,
attrs: attrsOf(selected.el),
outerHTML: (selected.el.outerHTML || '').slice(0, 2000),
},
candidates: ranked.slice(0, 10).map((row) => ({
index: row.index,
score: row.score,
statusHits: row.statusHits,
disabled: row.disabled,
text: row.text,
label: row.label,
attrs: attrsText(row.el),
outerHTML: (row.el.outerHTML || '').slice(0, 1000),
})),
};
}
selected.el.setAttribute('data-crawler-listing-status-dropdown', 'true');
const clickResult = clickTrigger(selected.el);
if (!clickResult.ok) {
return {
ok: false,
reason: 'listing status dropdown trigger click failed',
selected: {
index: selected.index,
score: selected.score,
text: selected.text,
label: selected.label,
attrs: attrsText(selected.el),
outerHTML: (selected.el.outerHTML || '').slice(0, 2000),
},
clickResult,
candidates: ranked.slice(0, 10).map((row) => ({
index: row.index,
score: row.score,
text: row.text,
label: row.label,
attrs: attrsText(row.el),
outerHTML: (row.el.outerHTML || '').slice(0, 1000),
})),
};
}
return {
ok: true,
selected: {
index: selected.index,
score: selected.score,
statusHits: selected.statusHits,
text: selected.text,
label: selected.label,
attrs: attrsText(selected.el),
outerHTML: (selected.el.outerHTML || '').slice(0, 2000),
},
trigger: clickResult.trigger,
candidates: ranked.slice(0, 10).map((row) => ({
index: row.index,
score: row.score,
statusHits: row.statusHits,
disabled: row.disabled,
text: row.text,
label: row.label,
attrs: attrsText(row.el),
outerHTML: (row.el.outerHTML || '').slice(0, 1000),
})),
};

View File

@@ -1,129 +0,0 @@
function textOf(node) {
return (node && (node.innerText || node.textContent || '') || '').replace(/\s+/g, ' ').trim();
}
function attrsOf(el) {
const attrs = {};
if (!el || !el.attributes) return attrs;
for (const attr of el.attributes) attrs[attr.name] = attr.value;
return attrs;
}
function isVisible(el) {
if (!el || !el.getBoundingClientRect) return false;
const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden';
}
function walkRoots(root, rows, seen) {
if (!root || seen.has(root)) return;
seen.add(root);
const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
let node = walker.currentNode;
while (node) {
if (node.nodeType === Node.ELEMENT_NODE && node.tagName) rows.push(node);
if (node.shadowRoot) walkRoots(node.shadowRoot, rows, seen);
node = walker.nextNode();
}
}
function allElements() {
const rows = [];
walkRoots(document, rows, new Set());
return rows;
}
function findMarkedDropdown() {
return allElements().find((el) => el.getAttribute('data-crawler-listing-status-dropdown') === 'true') || null;
}
function insideListingRow(el) {
return Boolean(el.closest('[data-sku], tr, [role="row"], [data-testid*="row"]'));
}
function normalize(text) {
return String(text || '').replace(/\s+/g, ' ').trim();
}
function isOptionLike(el) {
if (!el || !el.tagName) return false;
const tag = el.tagName.toLowerCase();
const role = el.getAttribute('role');
const className = el.getAttribute('class') || '';
return (
tag === 'kat-option' ||
tag === 'kat-label' ||
role === 'option' ||
/parentKatOptionStyle/i.test(className)
);
}
function parseStatus(rawText) {
const text = normalize(rawText);
if (!text) return '';
const match = text.match(/^(.*?)\s*[(]\s*[\d,]+\s*[)]\s*$/);
return normalize(match ? match[1] : text);
}
function isListingStatusOption(row) {
const text = normalize(`${row.raw_text} ${row.value || ''}`);
const parsed = parseStatus(row.raw_text);
const haystack = normalize(`${text} ${parsed}`);
if (/^(所有|SKUS?|ASIN|FNSKU|UPC\/EAN|商品名称\/关键字)$|^(ALL|SKU|ASIN|FNSKU|TITLE_KEYWORD|GTIN)$/i.test(haystack)) {
return false;
}
return /^(全部|在售|不可售|详情页面已删除|订单页面已删除|商品信息草稿|缺少的信息|定价问题|需要批准|配送问题|缺少报价|已停售|在搜索结果中禁止显示)$/.test(parsed);
}
function optionRow(el, index) {
const attrs = attrsOf(el);
const rawText = textOf(el) || attrs.label || attrs.text || attrs.value || '';
return {
index,
raw_text: rawText,
value: attrs.value || attrs['data-value'] || attrs.name || null,
tag: (el.tagName || '').toLowerCase(),
attrs,
visible: isVisible(el),
outerHTML: (el.outerHTML || '').slice(0, 1200),
};
}
const markedDropdown = findMarkedDropdown();
let scoped = [];
if (markedDropdown) {
const roots = [markedDropdown];
if (markedDropdown.shadowRoot) roots.push(markedDropdown.shadowRoot);
for (const root of roots) {
scoped = scoped.concat(Array.from(root.querySelectorAll('kat-option, kat-label, [role="option"], [class*="parentKatOptionStyle"]')));
}
}
const allOptions = allElements().filter((el) => {
if (!isOptionLike(el)) return false;
if (insideListingRow(el)) return false;
return isVisible(el);
});
const byNode = new Map();
const byStatus = new Map();
scoped.concat(allOptions).forEach((el, index) => {
if (!byNode.has(el)) {
const row = optionRow(el, index);
if (isListingStatusOption(row)) {
byNode.set(el, row);
const parsed = parseStatus(row.raw_text);
const hasValue = Boolean(row.value);
const hasQuantity = /[(]\s*[\d,]+\s*[)]/.test(row.raw_text);
const quality = (hasValue ? 2 : 0) + (hasQuantity ? 1 : 0);
const existing = byStatus.get(parsed);
if (!existing || quality > existing.quality) {
byStatus.set(parsed, { row, quality });
}
}
}
});
return Array.from(byStatus.values()).map((item) => item.row).filter((row) => row.raw_text);

View File

@@ -1,91 +0,0 @@
function textOf(node) {
return (node && (node.innerText || node.textContent || '') || '').replace(/\s+/g, ' ').trim();
}
function walkRoots(root, rows, seen) {
if (!root || seen.has(root)) return;
seen.add(root);
const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
let node = walker.currentNode;
while (node) {
if (node.nodeType === Node.ELEMENT_NODE && node.tagName) rows.push(node);
if (node.shadowRoot) walkRoots(node.shadowRoot, rows, seen);
node = walker.nextNode();
}
}
function allElements() {
const rows = [];
walkRoots(document, rows, new Set());
return rows;
}
function extractStatuses(text) {
const pattern = /(全部|在搜索结果中禁止显示|配送问题|缺少报价|已停售|详情页面已删除|订单页面已删除|商品信息草稿|缺少的信息|定价问题|需要批准|不可售|在售)\s*(?:[(]\s*([\d,]+)\s*[)])?/g;
const rows = [];
let match;
while ((match = pattern.exec(text)) !== null) {
rows.push({
status: match[1],
quantity: match[2] || '',
raw_text: match[0].trim(),
});
}
return rows;
}
function parseDropdown(dropdown, index) {
const nodes = Array.from(dropdown.querySelectorAll('kat-option, kat-label, [class*="parentKatOptionStyle"]'));
const seen = new Set();
const rows = nodes
.flatMap((node) => {
const rawText = textOf(node);
return extractStatuses(rawText).map((parsed) => ({
...parsed,
value: node.getAttribute('value') || '',
}));
})
.filter((row) => {
const key = `${row.status}::${row.quantity}`;
if (seen.has(key)) return false;
seen.add(key);
return true;
});
const quantityCount = rows.filter((row) => row.quantity).length;
const rawText = textOf(dropdown);
const className = String(dropdown.className || '');
const isStatusDropdown = /statusDropdown/i.test(className);
const score =
rows.length * 100 +
quantityCount * 50 +
(isStatusDropdown ? 500 : 0) +
(/在搜索结果中禁止显示|需要批准|详情页面已删除/.test(rawText) ? 200 : 0);
return {
index,
tag: (dropdown.tagName || '').toLowerCase(),
raw_text: rawText,
rows,
rowCount: rows.length,
quantityCount,
score,
className,
outerHTML: (dropdown.outerHTML || '').slice(0, 1200),
};
}
const ranked = allElements()
.map((el, index) => ({ el, index }))
.filter(({ el }) => (el.tagName || '').toLowerCase() === 'kat-dropdown')
.map(({ el, index }) => parseDropdown(el, index))
.filter((row) => row.rowCount >= 3)
.sort((a, b) => b.score - a.score);
const best = ranked[0];
return {
rows: best ? best.rows : [],
selected: best || null,
candidates: ranked.slice(0, 10),
};

View File

@@ -1,148 +0,0 @@
const __crawlerPayload = arguments[0] || {};
function textOf(node) {
return ((node && (node.innerText || node.textContent || '')) || '').replace(/\s+/g, ' ').trim();
}
function isVisible(el) {
if (!el || !el.getBoundingClientRect) return false;
const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden';
}
function walkRoots(root, rows, seen) {
if (!root || seen.has(root)) return;
seen.add(root);
const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
let node = walker.currentNode;
while (node) {
if (node.nodeType === Node.ELEMENT_NODE && node.tagName) rows.push(node);
if (node.shadowRoot) walkRoots(node.shadowRoot, rows, seen);
node = walker.nextNode();
}
}
function allElements() {
const rows = [];
walkRoots(document, rows, new Set());
return rows;
}
function findMarkedDropdown() {
return allElements().find((el) => el.getAttribute('data-crawler-listing-status-dropdown') === 'true') || null;
}
function insideListingRow(el) {
return Boolean(el.closest('[data-sku], tr, [role="row"], [data-testid*="row"]'));
}
function isOptionLike(el) {
if (!el || !el.tagName) return false;
const tag = el.tagName.toLowerCase();
const role = el.getAttribute('role');
const className = el.getAttribute('class') || '';
return (
tag === 'kat-option' ||
tag === 'kat-label' ||
role === 'option' ||
/parentKatOptionStyle/i.test(className)
);
}
function isSearchFieldOption(meta) {
return /^(所有|SKUS?|ASIN|FNSKU|UPC\/EAN|商品名称\/关键字)$|^(ALL|SKU|ASIN|FNSKU|TITLE_KEYWORD|GTIN)$/i.test(
`${meta.rawText} ${meta.parsedStatus} ${meta.value}`
);
}
function isListingStatusOption(meta) {
if (isSearchFieldOption(meta)) return false;
return /^(全部|在售|不可售|详情页面已删除|订单页面已删除|商品信息草稿|缺少的信息|定价问题|需要批准|配送问题|缺少报价|已停售|在搜索结果中禁止显示)$/.test(
meta.parsedStatus
);
}
function normalize(text) {
return String(text || '').replace(/\s+/g, ' ').trim();
}
function parseStatus(rawText) {
const text = normalize(rawText);
if (!text) return '';
const match = text.match(/^(.*?)\s*[(]\s*[\d,]+\s*[)]\s*$/);
return normalize(match ? match[1] : text);
}
function optionMeta(el, index) {
const rawText = textOf(el) || el.getAttribute('label') || el.getAttribute('text') || el.getAttribute('value') || '';
return {
index,
rawText: normalize(rawText),
parsedStatus: parseStatus(rawText),
value: el.getAttribute('value') || el.getAttribute('data-value') || '',
visible: isVisible(el),
};
}
function clickTargetFor(el) {
if (!el) return el;
return el.closest('[class*="parentKatOptionStyle"], kat-option, [role="option"]') || el;
}
const targetStatus = normalize((__crawlerPayload || {}).status);
const dropdown = findMarkedDropdown();
const options = [];
if (dropdown) {
const roots = [dropdown];
if (dropdown.shadowRoot) roots.push(dropdown.shadowRoot);
for (const root of roots) {
for (const el of Array.from(root.querySelectorAll('kat-option, kat-label, [role="option"], [class*="parentKatOptionStyle"]'))) {
options.push(el);
}
}
}
if (!options.length) {
for (const el of allElements()) {
if (!isOptionLike(el)) continue;
if (!isVisible(el) || insideListingRow(el)) continue;
options.push(el);
}
}
const metas = options
.map((el, index) => ({ el, ...optionMeta(el, index) }))
.filter((item) => isListingStatusOption(item));
const matched = metas.find((item) => item.parsedStatus === targetStatus || item.rawText === targetStatus);
if (!matched) {
return {
ok: false,
reason: 'status option not found',
targetStatus,
options: metas.map(({ index, rawText, parsedStatus, value, visible }) => ({
index,
rawText,
parsedStatus,
value,
visible,
})),
};
}
const clickTarget = clickTargetFor(matched.el);
clickTarget.scrollIntoView({ block: 'center', inline: 'nearest' });
clickTarget.click();
return {
ok: true,
targetStatus,
selected: {
index: matched.index,
rawText: matched.rawText,
parsedStatus: matched.parsedStatus,
value: matched.value,
clickedTag: clickTarget.tagName,
},
};

View File

@@ -1,94 +0,0 @@
function textOf(node) {
return (node && (node.innerText || node.textContent || '') || '').replace(/\s+/g, ' ').trim();
}
function attrsOf(el) {
const attrs = {};
if (!el || !el.attributes) return attrs;
for (const attr of el.attributes) attrs[attr.name] = attr.value;
return attrs;
}
function isVisible(el) {
if (!el || !el.getBoundingClientRect) return false;
const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden';
}
function walkRoots(root, rows, seen) {
if (!root || seen.has(root)) return;
seen.add(root);
const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
let node = walker.currentNode;
while (node) {
if (node.nodeType === Node.ELEMENT_NODE && node.tagName) rows.push(node);
if (node.shadowRoot) walkRoots(node.shadowRoot, rows, seen);
node = walker.nextNode();
}
}
function allElements() {
const rows = [];
walkRoots(document, rows, new Set());
return rows;
}
function snapshot(el, index) {
return {
index,
tag: el.tagName.toLowerCase(),
text: textOf(el),
attrs: attrsOf(el),
outerHTML: (el.outerHTML || '').slice(0, 2000),
};
}
const KEYWORD_RE = /推荐报价|featured offer|buy box|商城|marketplace|欧洲|英国|德国|法国|西班牙|意大利|2\s*天前|30\s*天前|2\s*days?\s*ago|30\s*days?\s*ago/i;
const visible = allElements().filter((el) => isVisible(el));
const keywordMatches = visible
.map((el, index) => ({ el, index, text: textOf(el) }))
.filter((row) => KEYWORD_RE.test(`${row.el.tagName} ${row.text} ${Object.values(attrsOf(row.el)).join(' ')}`))
.slice(0, 120)
.map(({ el, index }) => snapshot(el, index));
const buyBoxCard = visible.find((el) => (el.getAttribute('data-key') || '') === 'KPI_CARD_BUYBOX')
|| visible.find((el) => (el.id || '') === 'KPI_CARD_BUYBOX');
const buyBoxNeighborhood = [];
if (buyBoxCard) {
let node = buyBoxCard;
for (let depth = 0; node && depth < 6; depth += 1, node = node.parentElement) {
buyBoxNeighborhood.push({
depth,
tag: node.tagName.toLowerCase(),
text: textOf(node),
attrs: attrsOf(node),
outerHTML: (node.outerHTML || '').slice(0, 2500),
});
}
}
const buyBoxLeafMatches = [];
if (buyBoxCard) {
const scoped = [];
walkRoots(buyBoxCard, scoped, new Set());
scoped
.filter((el) => isVisible(el))
.map((el, index) => ({ el, index, text: textOf(el) }))
.filter((row) => row.text && row.text.length <= 200)
.filter((row) => /推荐报价|商城|marketplace|欧洲|英国|德国|法国|西班牙|意大利|\d+(?:\.\d+)?\s*%|2\s*天前|30\s*天前|2\s*days?\s*ago|30\s*days?\s*ago/i.test(row.text))
.slice(0, 120)
.forEach(({ el, index }) => buyBoxLeafMatches.push(snapshot(el, index)));
}
return {
url: window.location.href,
title: document.title,
buyBoxFound: !!buyBoxCard,
buyBoxNeighborhood,
buyBoxLeafMatches,
keywordMatches,
};

View File

@@ -1,132 +0,0 @@
function textOf(node) {
return (node && (node.innerText || node.textContent || '') || '').replace(/\s+/g, ' ').trim();
}
function attrsOf(el) {
const attrs = {};
if (!el || !el.attributes) return attrs;
for (const attr of el.attributes) attrs[attr.name] = attr.value;
return attrs;
}
function attrsText(el) {
return Object.entries(attrsOf(el)).map(([key, value]) => `${key}=${value}`).join(' ');
}
function isVisible(el) {
if (!el || !el.getBoundingClientRect) return false;
const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden';
}
function walkRoots(root, rows, seen) {
if (!root || seen.has(root)) return;
seen.add(root);
const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
let node = walker.currentNode;
while (node) {
if (node.nodeType === Node.ELEMENT_NODE && node.tagName) rows.push(node);
if (node.shadowRoot) walkRoots(node.shadowRoot, rows, seen);
node = walker.nextNode();
}
}
function allElements() {
const rows = [];
walkRoots(document, rows, new Set());
return rows;
}
function clickableFor(el) {
if (el && el.querySelector) {
const preferred = el.querySelector(
'casino-button[data-actionname="expand"], [data-actionname="expand"], casino-button[aria-label="chevron-down"], [aria-label="chevron-down"]'
);
if (preferred) return preferred;
}
let node = el;
for (let depth = 0; node && depth < 7; depth += 1, node = node.parentElement) {
if (node.querySelector) {
const preferred = node.querySelector(
'casino-button[data-actionname="expand"], [data-actionname="expand"], casino-button[aria-label="chevron-down"], [aria-label="chevron-down"]'
);
if (preferred) return preferred;
}
const tag = node.tagName.toLowerCase();
const role = node.getAttribute('role');
if (tag === 'button' || tag === 'a' || tag === 'kat-button' || role === 'button' || node.onclick) {
return node;
}
}
return el;
}
function scoreEntry(el) {
const text = textOf(el);
const haystack = `${el.tagName} ${attrsText(el)} ${text}`;
let score = 0;
if (/推荐报价百分比|推荐报价.*百分比|featured offer percentage|featured offer.*%|featured offer/i.test(haystack)) {
score += 120;
}
if (/buy box|购物车|报价/i.test(haystack)) score += 20;
if (/%/.test(haystack)) score += 15;
if (/button|link|href|click|card|metric/i.test(haystack)) score += 10;
if (isVisible(el)) score += 10;
if (text.length > 800) score -= 30;
return { score, text, haystack };
}
const ranked = allElements()
.filter((el) => isVisible(el))
.map((el, index) => ({ el, index, ...scoreEntry(el) }))
.filter((row) => row.score > 0)
.sort((a, b) => {
if (b.score !== a.score) return b.score - a.score;
return a.text.length - b.text.length;
});
if (!ranked.length) {
return {
ok: false,
reason: 'recommended offer percentage entry not found',
url: window.location.href,
title: document.title,
candidates: allElements()
.filter((el) => isVisible(el))
.map((el, index) => ({ index, tag: el.tagName.toLowerCase(), text: textOf(el), attrs: attrsOf(el) }))
.filter((row) => /推荐报价|featured offer|buy box|百分比|percentage/i.test(`${row.tag} ${row.text} ${Object.values(row.attrs).join(' ')}`))
.slice(0, 80),
};
}
const selected = ranked[0];
const clickable = clickableFor(selected.el);
clickable.setAttribute('data-crawler-recommended-offer-entry', 'true');
clickable.scrollIntoView({ block: 'center', inline: 'nearest' });
['pointerdown', 'mousedown', 'mouseup', 'click'].forEach((eventName) => {
clickable.dispatchEvent(new MouseEvent(eventName, { bubbles: true, cancelable: true, composed: true }));
});
if (typeof clickable.click === 'function') clickable.click();
return {
ok: true,
selected: {
index: selected.index,
score: selected.score,
text: selected.text,
attrs: attrsText(selected.el),
clickableTag: clickable.tagName.toLowerCase(),
clickableText: textOf(clickable),
outerHTML: (selected.el.outerHTML || '').slice(0, 2000),
},
candidates: ranked.slice(0, 12).map((row) => ({
index: row.index,
score: row.score,
text: row.text,
attrs: attrsText(row.el),
outerHTML: (row.el.outerHTML || '').slice(0, 1000),
})),
};

View File

@@ -1,332 +0,0 @@
function textOf(node) {
return (node && (node.innerText || node.textContent || '') || '').replace(/\s+/g, ' ').trim();
}
function attrsOf(el) {
const attrs = {};
if (!el || !el.attributes) return attrs;
for (const attr of el.attributes) attrs[attr.name] = attr.value;
return attrs;
}
function isVisible(el) {
if (!el || !el.getBoundingClientRect) return false;
const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden';
}
function walkRoots(root, rows, seen) {
if (!root || seen.has(root)) return;
seen.add(root);
const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
let node = walker.currentNode;
while (node) {
if (node.nodeType === Node.ELEMENT_NODE && node.tagName) rows.push(node);
if (node.shadowRoot) walkRoots(node.shadowRoot, rows, seen);
node = walker.nextNode();
}
}
function allElements() {
const rows = [];
walkRoots(document, rows, new Set());
return rows;
}
function rowOf(el, index) {
const attrs = attrsOf(el);
const rect = el.getBoundingClientRect();
return {
index,
tag: el.tagName.toLowerCase(),
raw_text: textOf(el),
label_text: attrs.label || attrs.title || attrs['aria-label'] || '',
percentage_text: attrs.value || attrs['data-value'] || '',
attrs,
visible: isVisible(el),
rect_left: rect.left,
rect_top: rect.top,
rect_width: rect.width,
rect_height: rect.height,
outerHTML: (el.outerHTML || '').slice(0, 1200),
};
}
const COUNTRIES = [
{ country: '欧洲', aliases: ['欧洲', 'Europe'] },
{ country: '英国', aliases: ['英国', 'UK', 'United Kingdom'] },
{ country: '德国', aliases: ['德国', 'Germany'] },
{ country: '法国', aliases: ['法国', 'France'] },
{ country: '西班牙', aliases: ['西班牙', 'Spain'] },
{ country: '意大利', aliases: ['意大利', 'Italy'] },
];
function percentagesOf(text) {
return (text.match(/\d+(?:\.\d+)?\s*%/g) || []).map((item) => item.replace(/\s+/g, ''));
}
function parentOrHost(el) {
if (!el) return null;
if (el.parentElement) return el.parentElement;
const root = el.getRootNode && el.getRootNode();
return root && root.host ? root.host : null;
}
function countryOfText(text) {
const normalized = (text || '').trim().toLowerCase();
return COUNTRIES.find(({ aliases }) => aliases.some((alias) => normalized === alias.toLowerCase())) || null;
}
function countCountries(text) {
return COUNTRIES.filter(({ aliases }) => aliases.some((alias) => new RegExp(`(^|\\s)${alias.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(\\s|$)`, 'i').test(text))).length;
}
function bestCountryRowText(el) {
const visited = new Set();
let node = el;
for (let depth = 0; node && depth < 7; depth += 1, node = parentOrHost(node)) {
if (visited.has(node)) break;
visited.add(node);
const text = textOf(node);
if (!text || text.length > 500) continue;
const percentages = percentagesOf(text);
if (percentages.length >= 2 && countCountries(text) <= 1) return text;
}
const parent = parentOrHost(el);
if (!parent || !parent.children) return '';
const children = Array.from(parent.children).filter((child) => isVisible(child) && textOf(child));
const index = children.indexOf(el);
if (index < 0) return '';
const nearby = children.slice(Math.max(0, index - 1), Math.min(children.length, index + 5)).map((child) => textOf(child));
const joined = nearby.join(' ');
return percentagesOf(joined).length >= 2 ? joined : '';
}
function directCountryRows(elements) {
const results = new Map();
elements.forEach((el) => {
const match = countryOfText(textOf(el));
if (!match || results.has(match.country)) return;
const text = bestCountryRowText(el);
const percentages = percentagesOf(text);
if (percentages.length < 2) return;
results.set(match.country, {
country: match.country,
ratio2DaysAgo: percentages[0],
ratio30DaysAgo: percentages[1],
raw_text: text,
source: 'rendered-country-row',
});
});
return Array.from(results.values());
}
function expandedCardOf(elements) {
return elements.find((el) => el.tagName && el.tagName.toLowerCase() === 'casino-card' && /\bcasino-expanded-card\b/.test(el.className || ''));
}
function directRowsFromExpandedCard(expandedCard) {
if (!expandedCard) return [];
const scoped = [];
walkRoots(expandedCard, scoped, new Set());
const countries = [];
const percentages = [];
const seen = new Set();
scoped
.filter((el) => isVisible(el))
.forEach((el) => {
const text = textOf(el);
const title = (el.getAttribute && el.getAttribute('title')) || '';
const aria = (el.getAttribute && (el.getAttribute('aria-label') || el.getAttribute('arialabel'))) || '';
const candidate = (title || aria || text || '').replace(/\s+/g, ' ').trim();
if (!candidate || candidate.length > 20) return;
const rect = el.getBoundingClientRect();
const area = rect.width * rect.height;
if (area <= 0 || area > 20000) return;
const key = `${candidate}|${Math.round(rect.left)}|${Math.round(rect.top)}|${Math.round(rect.width)}|${Math.round(rect.height)}`;
if (seen.has(key)) return;
seen.add(key);
const countryMatch = countryOfText(candidate);
if (countryMatch) {
countries.push({
country: countryMatch.country,
x: rect.left,
y: rect.top + rect.height / 2,
h: rect.height,
area,
});
return;
}
if (/^\d+(?:\.\d+)?\s*%$/.test(candidate)) {
percentages.push({
value: candidate.replace(/\s+/g, ''),
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2,
area,
});
}
});
const dedupedCountries = [];
countries
.sort((a, b) => a.y - b.y || a.x - b.x || a.area - b.area)
.forEach((item) => {
const exists = dedupedCountries.some((row) => row.country === item.country && Math.abs(row.y - item.y) < 8);
if (!exists) dedupedCountries.push(item);
});
const results = dedupedCountries.map((row) => {
const sameRow = percentages
.filter((item) => item.x > row.x + 20 && Math.abs(item.y - row.y) <= Math.max(14, row.h))
.sort((a, b) => a.x - b.x || a.y - b.y || a.area - b.area);
const unique = [];
sameRow.forEach((item) => {
if (!unique.some((existing) => existing.value === item.value && Math.abs(existing.x - item.x) < 8)) {
unique.push(item);
}
});
return {
country: row.country,
ratio2DaysAgo: unique[0] ? unique[0].value : '',
ratio30DaysAgo: unique[1] ? unique[1].value : '',
source: 'expanded-card-geometry',
};
});
return results.filter((item) => item.ratio2DaysAgo || item.ratio30DaysAgo);
}
function scoreRegion(el) {
const text = textOf(el);
if (!/%/.test(text)) return 0;
let score = 0;
if (/推荐报价百分比|推荐报价|featured offer percentage|featured offer|buy box/i.test(text)) score += 80;
if (/2\s*天前|30\s*天前|2\s*日前|30\s*日前|2\s*days?\s*ago|30\s*days?\s*ago/i.test(text)) score += 80;
if (/欧洲|英国|德国|法国|西班牙|意大利|Europe|UK|United Kingdom|Germany|France|Spain|Italy/i.test(text)) score += 40;
if (/%/.test(text)) score += 20;
if (isVisible(el)) score += 10;
if (text.length > 2500) score -= 40;
return score;
}
function isRelevantRatioRow(row) {
const combined = `${row.raw_text} ${row.label_text} ${row.percentage_text}`;
const hasDayLabels = /2\s*天前|30\s*天前|2\s*日前|30\s*日前|2\s*days?\s*ago|30\s*days?\s*ago/i.test(combined);
const hasCountries = /欧洲|英国|德国|法国|西班牙|意大利|Europe|UK|United Kingdom|Germany|France|Spain|Italy/i.test(combined);
if (!/%|2\s*天前|30\s*天前|2\s*日前|30\s*日前|2\s*days?\s*ago|30\s*days?\s*ago/i.test(combined) && !hasCountries) return false;
if (row.raw_text.length > 800 && !hasDayLabels && !hasCountries) return false;
return true;
}
function collectRelevantRows(sourceElements) {
const byText = new Map();
const scoped = [];
sourceElements.forEach((root) => walkRoots(root, scoped, new Set()));
scoped
.filter((el) => isVisible(el))
.forEach((el, index) => {
const row = rowOf(el, index);
if (!isRelevantRatioRow(row)) return;
const combined = `${row.raw_text} ${row.label_text} ${row.percentage_text}`;
const key = `${combined}|${Math.round(row.rect_left)}|${Math.round(row.rect_top)}|${Math.round(row.rect_width)}|${Math.round(row.rect_height)}`;
if (!byText.has(key)) byText.set(key, row);
});
return Array.from(byText.values());
}
function scrollableElements(elements) {
return elements.filter((el) => {
if (!el || !el.getBoundingClientRect || !isVisible(el)) return false;
const style = window.getComputedStyle(el);
if (!/(auto|scroll)/i.test(`${style.overflowY} ${style.overflow}`)) return false;
return el.scrollHeight > el.clientHeight + 20;
});
}
function collectRowsAcrossScroll(sourceElements, visibleElements) {
const targets = scrollableElements(visibleElements);
if (document.scrollingElement) targets.push(document.scrollingElement);
const byKey = new Map();
const originals = targets.map((target) => ({ target, top: target.scrollTop }));
for (const target of targets) {
const maxTop = Math.max(0, target.scrollHeight - target.clientHeight);
const positions = [...new Set([0, Math.floor(maxTop / 2), maxTop])];
for (const position of positions) {
target.scrollTop = position;
collectRelevantRows(sourceElements).forEach((row) => {
const combined = `${row.raw_text} ${row.label_text} ${row.percentage_text}`;
const key = `${combined}|${Math.round(row.rect_left)}|${Math.round(row.rect_top)}|${Math.round(row.rect_width)}|${Math.round(row.rect_height)}`;
if (!byKey.has(key)) byKey.set(key, row);
});
}
}
originals.forEach(({ target, top }) => {
target.scrollTop = top;
});
return Array.from(byKey.values());
}
const visible = allElements().filter((el) => isVisible(el));
const expandedCard = expandedCardOf(visible);
const expandedDirectRows = directRowsFromExpandedCard(expandedCard);
const directRows = expandedDirectRows.length ? expandedDirectRows : directCountryRows(visible);
const regions = visible
.map((el, index) => {
const rect = el.getBoundingClientRect();
return { el, index, score: scoreRegion(el), text: textOf(el), area: rect.width * rect.height };
})
.filter((row) => row.score > 0)
.sort((a, b) => {
if (b.score !== a.score) return b.score - a.score;
if (a.text.length !== b.text.length) return a.text.length - b.text.length;
return a.area - b.area;
});
const sourceElements = expandedCard ? [expandedCard] : (regions.length ? [regions[0].el] : visible);
const scoped = [];
sourceElements.forEach((root) => walkRoots(root, scoped, new Set()));
const byText = new Map();
collectRelevantRows(sourceElements)
.concat(collectRowsAcrossScroll(sourceElements, visible))
.forEach((row) => {
const combined = `${row.raw_text} ${row.label_text} ${row.percentage_text}`;
const key = `${combined}|${Math.round(row.rect_left)}|${Math.round(row.rect_top)}|${Math.round(row.rect_width)}|${Math.round(row.rect_height)}`;
if (!byText.has(key)) byText.set(key, row);
});
return {
directRows,
region: regions.length
? {
score: regions[0].score,
text: regions[0].text.slice(0, 2000),
outerHTML: (regions[0].el.outerHTML || '').slice(0, 3000),
}
: {
url: window.location.href,
title: document.title,
},
rows: Array.from(byText.values()),
candidates: regions.slice(0, 20).map((row) => ({
index: row.index,
score: row.score,
text: row.text.slice(0, 1000),
})),
};

View File

@@ -1,915 +0,0 @@
# encoding utf-8
import json
from pathlib import Path
import base64
import time
import re
import traceback
from datetime import datetime
from collections import defaultdict
import requests
from urllib.parse import quote
from curl_cffi import requests as requests_frp
from amazon.tool import show_notification,get_shop_info,remove_special_characters,split_currency_values
from amazon.amazon_base import TaskBase
from amazon.chrome_base import ChromeAmzoneBase
from config import runing_task, runing_shop,base_dir,DELETE_BRAND_API_BASE
try:
from config import proxy_url as CONFIG_PROXY_URL, proxy_mode as CONFIG_PROXY_MODE
except ImportError:
CONFIG_PROXY_URL = None
CONFIG_PROXY_MODE = 1
# Forbidden 后 3 分钟内统一使用代理:记录代理生效截止时间与当前代理
_FORBIDDEN_PROXY_UNTIL = 0.0
_FORBIDDEN_PROXY_DICT = None
_FORBIDDEN_PROXY_MINUTES = 0.5
class ChromeAmzone(ChromeAmzoneBase):
mark_name = "亚马逊相似ASIN"
def wait_load_complete(self):
try:
for _ in range(3):
content_area = self.tab.eles('xpath://div[@class="ap-sbi-aside__main"]', timeout=20)
if len(content_area) > 0:
pass
except Exception as e:
self.log(f"等待加载完成失败,{e},等待5秒")
time.sleep(5)
def encode_custom(self,input_):
if not input_ or not isinstance(input_, str):
return ""
transformed = ""
# 第一个字符:字符编码 + 字符串长度
transformed += chr((ord(input_[0]) + len(input_)) & 0xFFFF)
# 后续字符:当前字符编码 + 前一个字符编码
for i in range(1, len(input_)):
current_code = ord(input_[i])
prev_code = ord(input_[i - 1])
transformed += chr((current_code + prev_code) & 0xFFFF)
# 对应 JavaScript 的 encodeURIComponent
encoded = quote(transformed, safe="-_.!~*'()")
# 对应 replace(/[!'()*]/g, ...)
for ch in ["!", "'", "(", ")", "*"]:
encoded = encoded.replace(ch, "%" + format(ord(ch), "x"))
return encoded
def image_to_base64(self,image_source: str) -> str:
"""
将图片链接URL 或本地文件路径)转换为 Base64 编码的字符串。
"""
if image_source.startswith(('http://', 'https://')):
response = requests.get(image_source, timeout=10)
response.raise_for_status() # 非 2xx 状态码将抛出异常
image_data = response.content
else:
path = Path(image_source)
if not path.is_file():
raise ValueError(f"本地文件不存在: {image_source}")
with open(path, 'rb') as f:
image_data = f.read()
base64_str = base64.b64encode(image_data).decode('utf-8')
return base64_str
def get_aliprice_data(self,page=1,title="",domain="",category="",imageBase64=""):
"""
"fishkeeper Quick Aquarium Siphon Pump Gravel Cleaner - 256GPH Adjustable Powerful Fish Tank Vacuum Gravel Cleaning Kit for Aquarium Water Changer, Sand Cleaner, Dirt Removal : Amazon.co.uk: Pet Supplies"
标题 :
"""
global _FORBIDDEN_PROXY_UNTIL, _FORBIDDEN_PROXY_DICT
headers = {
"accept": "application/json, text/plain, */*",
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
"browser": "chrome",
"cache-control": "no-cache",
"channel": "chrome_offline",
"content-type": "application/json;charset=UTF-8",
"ext-id": "10100",
"ext_id": "10100",
"origin": "chrome-extension://ephkdklmdkaakeleplfpahjphaokcllh",
"platform": "1688",
"pragma": "no-cache",
"priority": "u=1, i",
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "none",
"sec-fetch-storage-access": "active",
"version": "3.7.4",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
}
cookie = {
"e-info": "[{\"e-name\":\"1688\",\"adid\":\"100\",\"version\":\"3.7.4\",\"ext_id\":\"10100\"}]",
"language": "chinese",
"province_code": "Guangdong",
"_ct": "czg",
"_e_ct": "czg",
"is_reto": "1",
"crossborder": "1",
"agent": "0",
"first_view": "1",
"currency": "USD",
"is_coo": "1",
"plugin": "chrome",
"plugin_ext": "100,129",
"m-info": "[{\"platform\":\"1688\",\"version\":\"3.7.4\",\"browser\":\"chrome\",\"m\":\"uni\",\"t\":1777902599578}]"
}
url = "https://api.aliprice.com/index.php/chrome/items/imageAnalysis"
itemTitle = f"{title} : {domain}: {category}"
params = {"page": page, "size": 20, "website": "1688_lite2", "language": "zh-CN", "currency": "USD", "from": "",
"itemTitle": itemTitle, "domain": domain }
params['sign'] = self.encode_custom(json.dumps(params, ensure_ascii=False))
data = {
"imageBase64" : imageBase64
}
proxies = None
# 若 3 分钟内曾出现 Forbidden则直接使用当时保存的代理
now = time.time()
if now < _FORBIDDEN_PROXY_UNTIL and _FORBIDDEN_PROXY_DICT:
proxies = _FORBIDDEN_PROXY_DICT
print("处于 Forbidden 代理窗口内,直接使用代理", proxies)
# 发送第一次请求
try:
response = requests_frp.post(url, headers=headers, params=params, json=data, impersonate="chrome101",
cookies=cookie, proxies=proxies)
response.encoding = "utf-8"
# 检查响应状态码和数据有效性
need_retry = False
if response.status_code != 200:
print(f"请求失败,状态码: {response.status_code}")
need_retry = True
else:
try:
result_data = response.json()
# 检查是否获取到有效数据
if not result_data or not isinstance(result_data, dict):
print("返回数据为空或格式不正确")
need_retry = True
elif "Forbidden" in response.text or "Too Many Requests" in response.text:
print("返回Forbidden或Too Many Requests")
need_retry = True
except Exception as e:
print(f"解析响应JSON失败: {e}")
need_retry = True
# 如果需要重试且配置了代理URL
if need_retry and CONFIG_PROXY_URL and not proxies:
try:
proxy_resp = requests.get(CONFIG_PROXY_URL, timeout=10)
print("代理请求结果->", proxy_resp.text)
# 模式 2账号密码代理接口返回 JSON
if CONFIG_PROXY_MODE == 2:
resp_json = proxy_resp.json()
proxy_list = resp_json.get("data", {}).get("list") or []
first_item = proxy_list[0] if proxy_list else None
if first_item:
ip = first_item.get("ip")
port = first_item.get("port")
account = first_item.get("account")
password = first_item.get("password")
if ip and port and account and password:
auth_proxy = f"{account}:{password}@{ip}:{port}"
print("获取到账号密码代理:", auth_proxy)
proxies = {
"http": f"http://{auth_proxy}",
"https": f"http://{auth_proxy}",
}
# 默认模式 1普通 IP:port 文本
if CONFIG_PROXY_MODE != 2:
proxy_ip = (proxy_resp.text or "").strip()
if proxy_ip:
proxies = {
"http": f"http://{proxy_ip}",
"https": f"http://{proxy_ip}",
}
if proxies:
print("使用代理重试请求")
response = requests_frp.post(url, headers=headers, params=params, json=data,
impersonate="chrome101", cookies=cookie, proxies=proxies)
response.encoding = "utf-8"
print("代理重试结果状态码:", response.status_code)
# 记录 3 分钟内都使用该代理
_FORBIDDEN_PROXY_UNTIL = now + _FORBIDDEN_PROXY_MINUTES * 60
_FORBIDDEN_PROXY_DICT = proxies
except Exception as e:
print("获取代理或重试失败:", e)
# 返回最终结果
result_data = response.json()
return result_data
except Exception as e:
print(f"get_aliprice_data 请求异常: {e}")
return {}
def _scrape_data(self):
"""
抓取商品数据
Returns:
dict: 抓取到的数据
"""
data = {
'image_url': "",
'title': "",
'category' : '',
'success' : True
}
try:
# 等待页面加载
# time.sleep(3)
title_ele = self.tab.ele('xpath://h1[@id="title"]',timeout=30)
title = title_ele.text
data["title"] = title
imge_ele = self.tab.ele('xpath://div[@id="imgTagWrapperId"]//img',timeout=20)
image_url = imge_ele.attr("src")
data["image_url"] = image_url
category = self.tab.eles('xpath://div[@id="wayfinding-breadcrumbs_feature_div"]//span[@class="a-list-item"]//a[@class="a-link-normal a-color-tertiary"]',timeout=20)
if len(category) > 0:
data["category"] = category[0].text
return data
except Exception as e:
print(f"抓取数据时出错: {traceback.format_exc()}")
data["success"] = False
return data
def run(self, country, asin,total_page=2):
"""
运行亚马逊详情采集任务
Args:
country: 国家名称(如:英国、德国、法国、西班牙、意大利)
asin: 亚马逊商品ASIN码
Returns:
dict: 包含采集到的数据
"""
try:
# 验证国家是否支持
if country not in self.country_info:
error_msg = f"不支持的国家: {country},支持的国家有: {list(self.country_info.keys())}"
print(error_msg)
show_notification(error_msg, "error")
return None
# 获取国家配置
country_config = self.country_info[country]
zip_code = country_config["zip_code"]
mark = country_config.get("mark")
# 1. 根据国家和ASIN拼接链接
base_url = country_config["url"]
# 提取域名部分
domain = base_url.split("/dp/")[0]
# 拼接新的URL
product_url = f"{domain}/dp/{asin}"
print(f"正在访问: {product_url}")
# 打开链接
self.tab.get(product_url)
time.sleep(3) # 等待页面初步加载
self.tab.wait.doc_loaded(timeout=30, raise_err=False)
self.close_init_popup()
# 2. 切换国家/设置邮编
print(f"正在检查并设置邮编: {zip_code},标识: {mark}")
self._set_zip_code(zip_code, mark)
# self.tab.wait.doc_loaded(timeout=5, raise_err=False)
# 3. 抓取数据
print("正在抓取商品数据...")
data = self._scrape_data()
# 判断是否采集标题出错
new_size = "220,220"
pattern = r"\._(?:[A-Z]+)?(\d+)_\."
replacement = f"._{new_size}_.jpg"
# 使用正则替换
image_new_url = re.sub(pattern, lambda m: replacement, data["image_url"])
iamge_base64 = self.image_to_base64(image_new_url)
similar_data = []
# 4、请求获取插件的数据
for page_num in range(total_page):
resp_data = self.get_aliprice_data(
page=page_num,
title = data["title"],
domain=domain,
category=data["category"],
imageBase64 = iamge_base64
)
self.log(f"Aliprice 扩展数据获取:{resp_data}")
if len(resp_data.get("data",[])) > 0:
for i in resp_data.get("data"):
similar_data.append(i)
data["similar_data"] = similar_data
# 添加基本信息
data['country'] = country
data['asin'] = asin
data['url'] = product_url
data['timestamp'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(f"数据抓取完成: {json.dumps(data)}")
return data
except Exception as e:
error_msg = f"运行出错: {traceback.format_exc()}"
print(error_msg)
show_notification(f"采集失败: {str(e)}", "error")
return {}
class SimilarAsinTask(TaskBase):
mark_name = "相似ASIN采集"
@staticmethod
def group_by_id_prefix(data):
"""
根据每条数据的 id 字段进行分组:
- 如果 id 为 "2_1",则取 "_" 前面的 "2" 作为分组 key
- 如果 id 为 "1",则分组 key 就是 "1"
- 相同 key 的数据归为同一组
:param data: 原始列表数据
:return: 二维列表,按 id 前缀分组
"""
grouped = defaultdict(list)
for item in data:
item_id = str(item.get("id", ""))
group_key = item_id.split("_")[0]
grouped[group_key].append(item)
return list(grouped.values())
@staticmethod
def normalize_groups(data):
groups = data.get("groups")
if isinstance(groups, list) and len(groups) > 0:
return groups
rows = data.get("rows") or data.get("items") or []
if not isinstance(rows, list) or len(rows) == 0:
return []
normalized = []
for items in SimilarAsinTask.group_by_id_prefix(rows):
first = items[0] if items else {}
item_id = str(first.get("id") or first.get("displayId") or "")
base_id = item_id.split("_")[0] if item_id else ""
normalized.append({
"sourceFileKey": first.get("sourceFileKey", ""),
"sourceFilename": first.get("sourceFilename", ""),
"groupKey": first.get("groupKey") or base_id,
"baseId": first.get("baseId") or base_id,
"displayId": first.get("displayId") or item_id,
"items": items,
})
return normalized
def fetch_parsed_payload(self, task_id, user_id=1):
url = f"{DELETE_BRAND_API_BASE}/api/similar-asin/tasks/{task_id}/parsed-payload"
response = requests.get(url, params={"user_id": user_id}, timeout=300, verify=False)
response.raise_for_status()
payload = response.json() if response.text else {}
if isinstance(payload, dict) and payload.get("success"):
return payload.get("data") or {}
raise RuntimeError(f"获取货源查询解析载荷失败: {payload}")
@staticmethod
def _count_payload_rows(data):
rows = data.get("rows") or data.get("items") or []
if isinstance(rows, list) and rows:
return len(rows)
groups = data.get("groups") or []
if not isinstance(groups, list):
return 0
total = 0
for group in groups:
if isinstance(group, dict) and isinstance(group.get("items"), list):
total += len(group.get("items"))
return total
def _merge_parsed_payload(self, data, parsed_payload):
payload_rows = parsed_payload.get("allItems") or parsed_payload.get("items") or []
merged = {
**data,
"groups": parsed_payload.get("groups") or [],
"rows": payload_rows,
}
if parsed_payload.get("aiPrompt"):
merged["prompt"] = parsed_payload.get("aiPrompt")
if parsed_payload.get("apiKey") and not merged.get("api_key"):
merged["api_key"] = parsed_payload.get("apiKey")
return merged
def process_task(self, task_data: dict):
"""处理审批任务主入口
Args:
task_data: 任务数据
"""
try:
data = task_data.get("data", {})
task_id = data.get("taskId")
if task_id:
queued_rows = self._count_payload_rows(data)
try:
parsed_payload = self.fetch_parsed_payload(task_id, data.get("user_id") or data.get("userId") or 1)
data = self._merge_parsed_payload(data, parsed_payload)
self.log(
f"similar asin task {task_id} loaded full parsed payload from Java, queuedRows={queued_rows}, fullRows={self._count_payload_rows(data)}"
)
except Exception as e:
if not queued_rows:
raise
self.log(
f"similar asin task {task_id} failed to load full parsed payload, fallback to queued rows={queued_rows}: {e}",
"WARNING"
)
groups = self.normalize_groups(data)
if not task_id:
self.log("任务ID为空跳过", "WARNING")
return
self.log(f"开始处理爬取任务 {task_id}{len(groups)} 个任务")
if not groups:
self.log("appearance-patent groups/rows is empty, skip", "WARNING")
return
runing_task[task_id] = {
"status": "running",
"start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"total_shops": 1,
"processed_shops": 0,
"total_countries": len(groups) ,
"processed_countries": 0,
"total_asins": 0,
"processed_asins": 0,
"success_count": 0,
"failed_count": 0,
"stop_requested": False
}
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理", "WARNING")
runing_task[task_id]["status"] = "stopped"
return
max_retry = 3
show_notification(f"开始爬取数据", "info")
chrome = ChromeAmzone()
try:
# 数据整理
# new_items = self.group_by_id_prefix(items)
result = []
for gp_index,gp in enumerate(groups):
items = gp.get("items", [])
return_data = {}
for index,value in enumerate(items):
return_data = {
'image_url': "",
'title': ""
}
asin = value.get("asin")
country = value.get("country")
return_data = None
for _ in range(max_retry):
try:
return_data = chrome.run(country, asin) or {}
self.log(f"抓取结果->{return_data}")
break
except Exception as e:
if "与页面的连接已断开" in str(e):
chrome = ChromeAmzone()
if not isinstance(return_data, dict):
return_data = {}
if return_data.get("image_url"):
break
group_item = [
{
"sourceFileKey": i.get("sourceFileKey"),
"sourceFilename": i.get("sourceFilename"),
"rowToken": i.get("rowToken"),
"groupKey": i.get("groupKey"),
"id": i.get("id"),
"asin": i.get("asin"),
"country": i.get("country"),
"url": return_data.get("image_url"),
"title": return_data.get("title"),
"done": False,
"urls" : [i.get("ori_picture") for i in return_data.get("similar_data")]
}
for i in items
]
# task_id: int, chunkIndex:int,chunkTotal: int, country_code: str, asin: str, status: dict,error:str="",
# item_data:dict={},
res = {
"sourceFileKey": gp.get("sourceFileKey"),
"sourceFilename": gp.get("sourceFilename"),
"groupKey": gp.get("groupKey"),
"baseId": gp.get("baseId"),
"displayId": gp.get("displayId"),
"items": group_item
}
print("================")
result.append(res)
print("================")
is_done = gp_index == len(groups)-1
if len(result) > 20 or is_done:
self.post_result(task_id=task_id,chunkIndex=gp_index+1,chunkTotal=len(groups),
asin=asin,item_data=result,is_done=is_done)
result = []
if len(result) > 0:
is_done = True
self.post_result(task_id=task_id, chunkIndex=len(groups), chunkTotal=len(groups),
asin=asin, item_data=result, is_done=is_done)
try:
chrome.close()
except Exception as e:
print("退出浏览器出错",e)
# 更新已处理店铺数
if task_id in runing_task:
runing_task[task_id]["processed_shops"] += 1
except Exception as e:
self.log(f"处理店铺 {task_id} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 更新任务状态
if task_id in runing_task:
if runing_task[task_id].get("stop_requested", False):
runing_task[task_id]["status"] = "stopped"
self.log(f"任务 {task_id} 已被暂停!")
else:
runing_task[task_id]["status"] = "completed"
self.log(f"任务 {task_id} 处理完成!")
except Exception as e:
self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR")
if task_id:
if task_id in runing_task:
runing_task[task_id]["status"] = "failed"
runing_task[task_id]["error"] = str(e)
def post_result(self, task_id: int, chunkIndex:int,chunkTotal: int, asin: str, error:str="",
item_data:dict={},
is_done: bool = False):
"""回传处理结果到API
"""
url = f"{DELETE_BRAND_API_BASE}/api/similar-asin/tasks/{task_id}/result"
payload ={
"submissionId": f"{task_id}",
"chunkIndex": chunkIndex,
"chunkTotal": chunkTotal,
"error": error,
"groups": item_data,
"done": is_done
}
max_retries = 3
for retry in range(max_retries):
try:
request_timeout = 300 if is_done else 30
print("================【详情采集】=====================")
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
self.log(f"回传URL: {url}")
self.log(f"回传数据: {payload}")
response = requests.post(
url,
json=payload,
headers={"Content-Type": "application/json"},
timeout=request_timeout,
verify=False
)
self.log(f"回传结果: {response.text}")
data = response.json() if response.text else {}
if response.status_code == 200 and isinstance(data, dict) and data.get("success"):
self.log(f"结果回传成功: {asin} - {item_data}")
return
else:
self.log(f"结果回传失败,状态码: {response.status_code}", "WARNING")
print("=====================================")
except Exception as e:
self.log(f"调用API异常: {str(e)}", "ERROR")
print("=====================================")
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(2)
self.log(f"已达到最大重试次数,结果回传最终失败", "ERROR")
raise RuntimeError("已达到最大重试次数,结果回传最终失败")
if __name__ == '__main__':
spide = ChromeAmzone()
resp = spide.run(
country="英国", asin="B0F79LCB3X", total_page=2
)
print(resp)
# task_data = {
# "type": "similar-asin-run",
# "ts": 1778038688824,
# "data": {
# "taskId": 6850,
# "api_key": "sk-EMcDFg36zCeWUtCbRzKUbrRZyNeC6M4KBhY6fAVcNP7GG4xI",
# "groups": [
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "groupKey": "similar-asin:6850",
# "baseId": "",
# "displayId": "1",
# "items": [
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::2",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::1@2",
# "id": "1",
# "asin": "B0D792ND9V",
# "country": "英国",
# "price": "12.29",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::3",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
# "id": "2_1",
# "asin": "B0D14L34S7",
# "country": "英国",
# "price": "14.88",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::4",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
# "id": "2_2",
# "asin": "B0D14N8FJY",
# "country": "英国",
# "price": "15.36",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::5",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
# "id": "2_3",
# "asin": "B0D14NBL5D",
# "country": "英国",
# "price": "14.87",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::6",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
# "id": "2_4",
# "asin": "B0D14M5X4J",
# "country": "英国",
# "price": "14.41",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::7",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::3@7",
# "id": "3_1",
# "asin": "B0CWQFZMGY",
# "country": "英国",
# "price": "17.05",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::8",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::3@7",
# "id": "3_2",
# "asin": "B0CX8D68L3",
# "country": "英国",
# "price": "16.46",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::9",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::4@9",
# "id": "4_1",
# "asin": "B0CX87XXWJ",
# "country": "英国",
# "price": "17.54",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::10",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::5@10",
# "id": "5_1",
# "asin": "B0CR3PMR8L",
# "country": "英国",
# "price": "16.74",
# "url": "",
# "title": "",
# "target_urls": []
# }
# ]
# }
# ],
# "rows": [
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::2",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::1@2",
# "id": "1",
# "asin": "B0D792ND9V",
# "country": "英国",
# "price": "12.29",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::3",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
# "id": "2_1",
# "asin": "B0D14L34S7",
# "country": "英国",
# "price": "14.88",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::4",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
# "id": "2_2",
# "asin": "B0D14N8FJY",
# "country": "英国",
# "price": "15.36",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::5",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
# "id": "2_3",
# "asin": "B0D14NBL5D",
# "country": "英国",
# "price": "14.87",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::6",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
# "id": "2_4",
# "asin": "B0D14M5X4J",
# "country": "英国",
# "price": "14.41",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::7",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::3@7",
# "id": "3_1",
# "asin": "B0CWQFZMGY",
# "country": "英国",
# "price": "17.05",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::8",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::3@7",
# "id": "3_2",
# "asin": "B0CX8D68L3",
# "country": "英国",
# "price": "16.46",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::9",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::4@9",
# "id": "4_1",
# "asin": "B0CX87XXWJ",
# "country": "英国",
# "price": "17.54",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::10",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::5@10",
# "id": "5_1",
# "asin": "B0CR3PMR8L",
# "country": "英国",
# "price": "16.74",
# "url": "",
# "title": "",
# "target_urls": []
# }
# ]
# }
# }
# sim_asin = SimilarAsinTask({})
# sim_asin.process_task(task_data)

View File

@@ -1,161 +0,0 @@
# 导入 webview 用于前端通知
try:
import webview
except ImportError:
webview = None
import requests
from urllib.parse import quote
import re
def show_notification(message: str, message_type: str = "error"):
"""显示 pywebview 顶层通知5秒后自动消失
Args:
message: 通知消息
message_type: 消息类型 (success/warning/error/info)
"""
if webview and webview.windows:
try:
# 转义单引号,防止 JavaScript 语法错误
safe_message = message.replace("'", "\\'").replace('"', '\\"').replace('\n', '\\n')
# 设置通知样式颜色
color_map = {
'success': '#67C23A',
'warning': '#E6A23C',
'error': '#F56C6C',
'info': '#909399'
}
bg_color = color_map.get(message_type, '#F56C6C')
# 构建前端通知的 JavaScript 代码 - 创建原生 HTML 通知
js_code = f"""
(function() {{
// 移除已存在的通知
var existingNotif = document.getElementById('pywebview-notification');
if (existingNotif) {{
existingNotif.remove();
}}
// 创建通知容器
var notif = document.createElement('div');
notif.id = 'pywebview-notification';
notif.style.cssText = 'position: fixed; top: 20px; left: 50%; transform: translateX(-50%); ' +
'background: {bg_color}; color: white; padding: 12px 20px; border-radius: 4px; ' +
'box-shadow: 0 2px 12px rgba(0,0,0,0.3); z-index: 99999; font-size: 14px; ' +
'max-width: 600px; word-wrap: break-word; display: flex; align-items: center; gap: 10px;';
// 添加消息内容
var msgSpan = document.createElement('span');
msgSpan.textContent = '{safe_message}';
notif.appendChild(msgSpan);
// 添加关闭按钮
var closeBtn = document.createElement('span');
closeBtn.innerHTML = '×';
closeBtn.style.cssText = 'cursor: pointer; font-size: 18px; font-weight: bold; margin-left: 10px;';
closeBtn.onclick = function() {{ notif.remove(); }};
notif.appendChild(closeBtn);
// 添加到页面
document.body.appendChild(notif);
// 5秒后自动消失
setTimeout(function() {{
if (notif && notif.parentNode) {{
notif.style.transition = 'opacity 0.3s';
notif.style.opacity = '0';
setTimeout(function() {{ notif.remove(); }}, 300);
}}
}}, 5000);
}})();
"""
# 在第一个窗口中执行 JavaScript
webview.windows[0].evaluate_js(js_code)
except Exception as e:
print(f"显示通知失败: {str(e)}")
def get_shop_info(shop_name: str, base_url: str = "http://8.136.19.173:18080") -> requests.Response:
"""
调用商铺凭证接口
:param shop_name: 商铺名称,将自动进行 URL 编码
:param base_url: API 基础地址,默认从 curl 中提取
:return: requests.Response 对象
"""
url = f"{base_url}/api/admin/shop-manages/credential"
params = {"shopName": shop_name}
headers = {
"Accept": "*/*",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"Pragma": "no-cache",
"Referer": "http://8.136.19.173:18080/doc.html",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
"X-Internal-Token": "59c6691199917a2095827b5188502029",
}
response = requests.get(
url,
params=params,
headers=headers,
verify=False,
timeout=30
)
return response
def remove_special_characters(text: str) -> str:
"""
去除字符串中的特殊字符,只保留数字、小数点和负号。
例如:'£24.55' -> '24.55'
"""
# 匹配所有允许的字符:数字、小数点、负号
# 注意:负号必须位于开头才合法,但这里只做字符保留,不做格式校验
cleaned = re.sub(r'[^0-9.-]', '', text)
return cleaned
def split_currency_values(currency_str: str) -> tuple[float, float]:
"""
将包含两个货币值的字符串拆分成两个浮点数。
参数:
currency_str (str): 格式如 "€22.64 + €0.00" 的字符串,中间以 '+' 分隔,
每部分可包含任意货币符号或前缀/后缀。
返回:
tuple[float, float]: 两个数值,顺序与字符串中的出现顺序一致。
异常:
ValueError: 如果字符串不包含正好两个部分,或者任一部分中无法提取到数值。
"""
# 按第一个 '+' 分割,最多分为两部分
parts = currency_str.split('+', 1)
if len(parts) != 2:
raise ValueError("字符串必须包含两个由 '+' 分隔的部分")
# 匹配整数或浮点数(可选负号)
number_pattern = r'-?\d+(?:\.\d+)?'
values = []
for part in parts:
# 去除首尾空格
part = part.strip()
match = re.search(number_pattern, part)
if not match:
raise ValueError(f"无法从 '{part}' 中提取数值")
values.append(float(match.group()))
return tuple(values)
if __name__ == '__main__':
cu_str = "€58.44 + €0.00 匹配"
res = split_currency_values(cu_str)
print(sum(res))

View File

@@ -1,106 +0,0 @@
"""
卖相AI - Flask 后端
按功能拆分为蓝图:认证(auth)、主页面(main)、管理员(admin)、图片(image)、品牌(brand)
"""
import os
import secrets
import logging
from datetime import timedelta, datetime
from logging.handlers import RotatingFileHandler
from flask import Flask
from flask_cors import CORS
from app_common import init_db, BASE_DIR
from blueprints.auth import auth_bp
from blueprints.main import main_bp
from blueprints.admin import admin_bp
from blueprints.image import image_bp
from blueprints.brand import brand_bp
from blueprints.communication import communication_bp
from config import cache_path, debug
def setup_flask_logging(app):
"""配置Flask访问日志输出到独立的日志文件"""
api_cache_path = os.path.join(cache_path, "api_logs")
os.makedirs(api_cache_path, exist_ok=True)
# 只在非debug模式下配置文件日志
if not debug:
# 获取当前日期作为日志文件名的一部分
today = datetime.now().strftime("%Y_%m_%d")
log_file = os.path.join(api_cache_path, f'api_{today}.log')
# 创建文件处理器使用RotatingFileHandler避免日志文件过大
file_handler = RotatingFileHandler(
log_file,
maxBytes=10 * 1024 * 1024, # 10MB
backupCount=5,
encoding='utf-8'
)
# 设置日志格式
formatter = logging.Formatter(
'[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S'
)
file_handler.setFormatter(formatter)
file_handler.setLevel(logging.INFO)
# 配置Flask应用日志
app.logger.addHandler(file_handler)
app.logger.setLevel(logging.INFO)
# 配置Werkzeug日志HTTP访问日志
werkzeug_logger = logging.getLogger('werkzeug')
werkzeug_logger.addHandler(file_handler)
werkzeug_logger.setLevel(logging.INFO)
print(f"API 日志已配置,输出到: {log_file}")
else:
print("Debug模式下API 日志输出到控制台")
def create_app():
app = Flask(__name__, template_folder=BASE_DIR, static_folder=BASE_DIR)
frontend_origin = os.environ.get('FRONTEND_ORIGIN', '*')
cors_origins = frontend_origin if frontend_origin != '*' else '*'
CORS(
app,
supports_credentials=True,
resources={r"/api/*": {"origins": cors_origins}, r"/login": {"origins": cors_origins}},
)
# 生产环境必须通过环境变量注入固定 SECRET_KEY否则服务重启会使旧会话失效。
app.secret_key = os.environ.get('SECRET_KEY', 'dev-secret-key-change-me')
app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(days=7)
app.config['SESSION_COOKIE_HTTPONLY'] = True
app.config['SESSION_COOKIE_SAMESITE'] = os.environ.get('SESSION_COOKIE_SAMESITE', 'Lax')
app.config['SESSION_COOKIE_SECURE'] = os.environ.get('SESSION_COOKIE_SECURE', '0') == '1'
# 注册蓝图(不设 url_prefix保持原有 URL 路径不变,前端无需改动)
app.register_blueprint(auth_bp)
app.register_blueprint(main_bp)
app.register_blueprint(admin_bp)
app.register_blueprint(image_bp)
app.register_blueprint(brand_bp)
app.register_blueprint(communication_bp)
# 配置Flask访问日志
setup_flask_logging(app)
return app
app = create_app()
def run_app(host='127.0.0.1', port=5123):
# init_db()
app.run(host=host, port=port, threaded=True, use_reloader=False)
if __name__ == '__main__':
run_app()

View File

@@ -1,291 +0,0 @@
"""
公共模块:数据库连接、初始化、会话校验、装饰器、模板渲染
供各蓝图复用
"""
import os
import secrets
from datetime import timedelta
from functools import wraps
import pymysql
from flask import request, redirect, url_for, session, jsonify, render_template, render_template_string
from config import mysql_host, mysql_user, mysql_password, mysql_database
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
STATIC_DIR = os.path.join(BASE_DIR, 'static')
ASSETS_DIR = os.path.join(BASE_DIR, 'assets')
WEB_SOURCE_DIR = os.path.join(BASE_DIR, 'web_source')
TEMPLATE_FALLBACK_DIRS = (
WEB_SOURCE_DIR,
os.path.join(WEB_SOURCE_DIR, 'templates_backup'),
)
def get_db():
return pymysql.connect(
host=mysql_host,
user=mysql_user,
password=mysql_password,
database=mysql_database,
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor
)
def _render_html(template_name: str, **context):
"""读取 HTML 模板:若为加密文件则先解密,再渲染。未加密或解密失败时按明文渲染。"""
path = next(
(candidate for candidate in (os.path.join(base_path, template_name) for base_path in TEMPLATE_FALLBACK_DIRS)
if os.path.isfile(candidate)),
None,
)
if path is None:
return render_template(template_name, **context)
with open(path, "rb") as f:
raw = f.read()
try:
from html_crypto import decrypt
content = decrypt(raw).decode("utf-8")
except Exception:
content = raw.decode("utf-8", errors="replace")
return render_template_string(content, **context)
def _render_html_new(template_name: str, **context):
"""读取 HTML 模板:若为加密文件则先解密,再渲染。未加密或解密失败时按明文渲染。"""
path = next(
(candidate for candidate in (os.path.join(base_path, template_name) for base_path in TEMPLATE_FALLBACK_DIRS)
if os.path.isfile(candidate)),
None,
)
if path is None:
return render_template(template_name, **context)
with open(path, "rb") as f:
raw = f.read()
try:
from html_crypto import decrypt
content = decrypt(raw).decode("utf-8")
except Exception:
content = raw.decode("utf-8", errors="replace")
return render_template_string(content, **context)
def _is_session_user_valid():
"""校验 session 中的 user_id 是否在数据库中仍存在;不存在则清除 session 并返回 False"""
uid = session.get('user_id')
if not uid:
return False
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute("SELECT id FROM users WHERE id = %s", (uid,))
row = cur.fetchone()
conn.close()
if not row:
session.clear()
return False
return True
except Exception:
session.clear()
return False
def _get_current_admin_role():
"""获取当前登录用户的管理角色super_admin / admin / None非管理员"""
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute(
"SELECT id, username, is_admin, role, created_by_id FROM users WHERE id = %s",
(session['user_id'],)
)
row = cur.fetchone()
conn.close()
if not row or not row.get('is_admin'):
return None, None
return row.get('role') or ('super_admin' if row.get('created_by_id') is None else 'admin'), row
except Exception:
return None, None
def login_required(f):
@wraps(f)
def decorated(*args, **kwargs):
if not session.get('user_id') or not _is_session_user_valid():
if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
return jsonify({'success': False, 'error': '未登录'}), 401
return redirect(url_for('auth.login'))
return f(*args, **kwargs)
return decorated
def admin_required(f):
@wraps(f)
def decorated(*args, **kwargs):
if not session.get('user_id'):
if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
return jsonify({'success': False, 'error': '未登录'}), 401
return redirect(url_for('auth.login'))
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute("SELECT is_admin, role FROM users WHERE id = %s", (session['user_id'],))
row = cur.fetchone()
conn.close()
if not row or not row.get('is_admin'):
if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
return jsonify({'success': False, 'error': '需要管理员权限'}), 403
return redirect(url_for('main.home'))
except Exception as e:
if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
return jsonify({'success': False, 'error': str(e)}), 500
return redirect(url_for('main.home'))
return f(*args, **kwargs)
return decorated
def init_db():
"""初始化数据库表,若不存在则创建"""
from werkzeug.security import generate_password_hash
conn = pymysql.connect(
host=mysql_host,
user=mysql_user,
password=mysql_password,
charset='utf8mb4'
)
try:
with conn.cursor() as cur:
cur.execute(f"CREATE DATABASE IF NOT EXISTS `{mysql_database}` DEFAULT CHARSET utf8mb4")
cur.execute(f"USE `{mysql_database}`")
cur.execute("""
CREATE TABLE IF NOT EXISTS users (
id INT AUTO_INCREMENT PRIMARY KEY,
username VARCHAR(64) NOT NULL UNIQUE,
password_hash VARCHAR(256) NOT NULL,
is_admin TINYINT(1) DEFAULT 0,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
""")
cur.execute("""
CREATE TABLE IF NOT EXISTS image_history (
id INT AUTO_INCREMENT PRIMARY KEY,
user_id INT NOT NULL,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
panel_type VARCHAR(64) DEFAULT '',
original_urls JSON,
params JSON,
result_urls JSON,
long_image_url VARCHAR(1024) DEFAULT NULL,
INDEX idx_user_created (user_id, created_at DESC)
)
""")
try:
cur.execute("ALTER TABLE image_history ADD COLUMN long_image_url VARCHAR(1024) DEFAULT NULL")
except Exception:
pass
try:
cur.execute("ALTER TABLE users ADD COLUMN machine VARCHAR(64) DEFAULT NULL")
except Exception:
pass
try:
cur.execute("ALTER TABLE users ADD COLUMN role VARCHAR(20) DEFAULT 'normal'")
except Exception:
pass
try:
cur.execute("ALTER TABLE users ADD COLUMN created_by_id INT NULL")
except Exception:
pass
cur.execute("""
CREATE TABLE IF NOT EXISTS brand_crawl_tasks (
id INT AUTO_INCREMENT PRIMARY KEY,
user_id INT NOT NULL,
file_paths JSON NOT NULL,
status VARCHAR(20) DEFAULT 'pending',
task_type TINYINT DEFAULT 1 COMMENT '1=立即执行,2=添加任务执行',
result_paths JSON NULL,
error_message TEXT NULL,
progress_current INT DEFAULT 0,
progress_total INT DEFAULT 0,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
INDEX idx_user_status (user_id, status)
)
""")
try:
cur.execute("ALTER TABLE brand_crawl_tasks ADD COLUMN progress_current INT DEFAULT 0")
except Exception:
pass
try:
cur.execute("ALTER TABLE brand_crawl_tasks ADD COLUMN progress_total INT DEFAULT 0")
except Exception:
pass
try:
cur.execute("ALTER TABLE brand_crawl_tasks ADD COLUMN `desc` VARCHAR(500) NULL COMMENT '上传文件名描述'")
except Exception:
pass
try:
cur.execute("ALTER TABLE brand_crawl_tasks ADD COLUMN strategy VARCHAR(20) DEFAULT 'Terms' COMMENT '品牌匹配方式: Terms/Simple'")
except Exception:
pass
try:
cur.execute("ALTER TABLE brand_crawl_tasks ADD COLUMN task_type TINYINT DEFAULT 1 COMMENT '1=立即执行,2=添加任务执行'")
except Exception:
pass
cur.execute("""
CREATE TABLE IF NOT EXISTS columns (
id INT AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(128) NOT NULL COMMENT '栏目名',
column_key VARCHAR(64) NOT NULL COMMENT '栏目标识',
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
UNIQUE KEY uk_column_key (column_key)
)
""")
cur.execute("""
CREATE TABLE IF NOT EXISTS user_column_permission (
user_id INT NOT NULL,
column_id INT NOT NULL,
PRIMARY KEY (user_id, column_id),
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE,
FOREIGN KEY (column_id) REFERENCES columns(id) ON DELETE CASCADE
)
""")
try:
cur.execute("UPDATE users SET role = 'normal' WHERE (role IS NULL OR role = '') AND (is_admin = 0 OR is_admin IS NULL)")
cur.execute("SELECT MIN(id) AS mid FROM users WHERE is_admin = 1")
row = cur.fetchone()
if row and row.get('mid'):
mid = row['mid']
cur.execute("UPDATE users SET role = 'super_admin' WHERE id = %s", (mid,))
cur.execute("UPDATE users SET role = 'admin' WHERE is_admin = 1 AND id != %s", (mid,))
cur.execute("UPDATE users SET created_by_id = %s WHERE role = 'admin' AND (created_by_id IS NULL)", (mid,))
except Exception:
pass
conn.commit()
finally:
conn.close()
_create_initial_admin()
def _create_initial_admin():
"""若没有任何管理员,则创建默认超级管理员(首次启动时,仅一个)"""
from werkzeug.security import generate_password_hash
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute("SELECT id FROM users WHERE role = 'super_admin' LIMIT 1")
if cur.fetchone():
conn.close()
return
admin_user = os.environ.get('ADMIN_USER', 'admin')
admin_pwd = os.environ.get('ADMIN_PASSWORD', 'admin123')
pwd_hash = generate_password_hash(admin_pwd, method='pbkdf2:sha256')
cur.execute(
"INSERT INTO users (username, password_hash, is_admin, role) VALUES (%s, %s, 1, 'super_admin')",
(admin_user, pwd_hash)
)
conn.commit()
conn.close()
except Exception:
pass

View File

@@ -1 +0,0 @@
# 蓝图包

View File

@@ -1,379 +0,0 @@
"""
管理员蓝图:用户管理(列表/创建/更新/删除)、生成历史、管理页
"""
import json
import pymysql
from flask import Blueprint, request, jsonify
from werkzeug.security import generate_password_hash
from app_common import get_db, _render_html, _get_current_admin_role, admin_required, login_required
from flask import session
admin_bp = Blueprint('admin', __name__)
def _parse_json(val, default=None):
if val is None:
return default if default is not None else []
if isinstance(val, (list, dict)):
return val
try:
return json.loads(val)
except Exception:
return default if default is not None else []
@admin_bp.route('/admin')
@login_required
@admin_required
def admin_page():
return _render_html('admin.html')
@admin_bp.route('/api/admin/users')
@admin_required
def admin_list_users():
"""分页获取用户列表;支持用户名模糊搜索、指定管理员所属普通用户筛选"""
role, current_row = _get_current_admin_role()
if not role:
return jsonify({'success': False, 'error': '需要管理员权限'}), 403
page = max(1, int(request.args.get('page', 1)))
page_size = min(50, max(5, int(request.args.get('page_size', 15))))
offset = (page - 1) * page_size
search_username = (request.args.get('username') or request.args.get('search') or '').strip()
created_by_id_arg = request.args.get('created_by_id') or request.args.get('admin_id')
created_by_id = int(created_by_id_arg) if created_by_id_arg and str(created_by_id_arg).isdigit() else None
if role != 'super_admin':
created_by_id = None
try:
conn = get_db()
with conn.cursor() as cur:
if role == 'super_admin':
where_parts = ["1=1"]
params = []
if search_username:
where_parts.append("u.username LIKE %s")
params.append("%" + search_username + "%")
if created_by_id is not None:
where_parts.append("u.created_by_id = %s")
params.append(created_by_id)
where_sql = " AND ".join(where_parts)
cur.execute(
"""SELECT u.id, u.username, u.is_admin, u.role, u.created_at, u.created_by_id,
creator.username AS creator_username
FROM users u
LEFT JOIN users creator ON creator.id = u.created_by_id
WHERE """ + where_sql + """ ORDER BY u.id LIMIT %s OFFSET %s""",
tuple(params) + (page_size, offset),
)
rows = cur.fetchall()
cur.execute("SELECT COUNT(*) as total FROM users u WHERE " + where_sql, tuple(params))
total = cur.fetchone()['total']
cur.execute("SELECT id, username FROM users WHERE role = 'admin' ORDER BY id")
admins = [{'id': r['id'], 'username': r['username']} for r in cur.fetchall()]
else:
admin_id = current_row['id']
where_parts = ["(u.id = %s OR (u.role = 'normal' AND u.created_by_id = %s))"]
params = [admin_id, admin_id]
if search_username:
where_parts.append("u.username LIKE %s")
params.append("%" + search_username + "%")
where_sql = " AND ".join(where_parts)
cur.execute(
"""SELECT u.id, u.username, u.is_admin, u.role, u.created_at, u.created_by_id,
creator.username AS creator_username
FROM users u
LEFT JOIN users creator ON creator.id = u.created_by_id
WHERE """ + where_sql + """ ORDER BY u.id LIMIT %s OFFSET %s""",
tuple(params) + (page_size, offset),
)
rows = cur.fetchall()
cur.execute(
"SELECT COUNT(*) as total FROM users u WHERE " + where_sql,
tuple(params),
)
total = cur.fetchone()['total']
admins = []
items = [
{
'id': r['id'],
'username': r['username'],
'is_admin': bool(r.get('is_admin')),
'role': r.get('role') or 'normal',
'created_by_id': r.get('created_by_id'),
'creator_username': r.get('creator_username') or '',
'created_at': r['created_at'].strftime('%Y-%m-%d %H:%M') if r.get('created_at') else '',
}
for r in rows
]
conn.close()
payload = {
'success': True,
'items': items,
'total': total,
'page': page,
'page_size': page_size,
'current_user_role': role,
'admins': admins,
}
return jsonify(payload)
except Exception as e:
return jsonify({'success': False, 'error': str(e)})
@admin_bp.route('/api/admin/user', methods=['POST'])
@admin_required
def admin_create_user():
data = request.get_json() or {}
username = (data.get('username') or '').strip()
password = data.get('password') or ''
role, current_row = _get_current_admin_role()
if not role:
return jsonify({'success': False, 'error': '需要管理员权限'}), 403
want_role = (data.get('role') or 'normal').strip() or 'normal'
if want_role not in ('admin', 'normal'):
want_role = 'normal'
if role == 'admin' and want_role == 'admin':
return jsonify({'success': False, 'error': '仅超级管理员可创建管理员'})
if want_role == 'admin':
want_created_by = current_row['id']
elif role == 'super_admin':
want_created_by = data.get('created_by_id')
else:
want_created_by = current_row['id']
if not username or not password:
return jsonify({'success': False, 'error': '用户名和密码不能为空'})
if len(username) < 2:
return jsonify({'success': False, 'error': '用户名至少2个字符'})
if len(password) < 6:
return jsonify({'success': False, 'error': '密码至少6个字符'})
if want_role == 'normal' and role == 'super_admin' and want_created_by is None:
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute("SELECT id FROM users WHERE role = 'admin' ORDER BY id LIMIT 1")
r = cur.fetchone()
conn.close()
want_created_by = r['id'] if r else current_row['id']
except Exception:
want_created_by = current_row['id']
if want_role == 'normal' and want_created_by is None:
want_created_by = current_row['id']
is_admin = 1 if want_role in ('super_admin', 'admin') else 0
pwd_hash = generate_password_hash(password, method='pbkdf2:sha256')
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute(
"INSERT INTO users (username, password_hash, is_admin, role, created_by_id) VALUES (%s, %s, %s, %s, %s)",
(username, pwd_hash, is_admin, want_role, want_created_by),
)
conn.commit()
conn.close()
return jsonify({'success': True, 'msg': '用户创建成功'})
except pymysql.IntegrityError:
return jsonify({'success': False, 'error': '用户名已存在'})
except Exception as e:
return jsonify({'success': False, 'error': str(e)})
@admin_bp.route('/api/admin/user/<int:uid>', methods=['PUT'])
@admin_required
def admin_update_user(uid):
data = request.get_json() or {}
password = data.get('password')
want_role = (data.get('role') or '').strip() or data.get('role')
role, current_row = _get_current_admin_role()
if not role:
return jsonify({'success': False, 'error': '需要管理员权限'}), 403
if want_role is None and not password:
return jsonify({'success': False, 'error': '请提供要修改的内容'})
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute("SELECT id, role, created_by_id FROM users WHERE id = %s", (uid,))
target = cur.fetchone()
if not target:
conn.close()
return jsonify({'success': False, 'error': '用户不存在'})
if role == 'admin':
if target['role'] != 'normal' or target.get('created_by_id') != current_row['id']:
conn.close()
return jsonify({'success': False, 'error': '只能编辑自己创建的普通用户'}), 403
want_role = None
else:
if target.get('role') == 'super_admin':
conn.close()
return jsonify({'success': False, 'error': '不能修改超级管理员'})
if want_role == 'super_admin':
return jsonify({'success': False, 'error': '不能将用户设为超级管理员'})
if want_role not in ('admin', 'normal', None, ''):
want_role = None
if password:
if len(password) < 6:
conn.close()
return jsonify({'success': False, 'error': '密码至少6个字符'})
pwd_hash = generate_password_hash(password, method='pbkdf2:sha256')
cur.execute("UPDATE users SET password_hash = %s WHERE id = %s", (pwd_hash, uid))
if want_role is not None and want_role != '':
is_admin = 1 if want_role == 'admin' else 0
cur.execute(
"UPDATE users SET is_admin = %s, role = %s WHERE id = %s",
(is_admin, want_role, uid),
)
conn.commit()
conn.close()
return jsonify({'success': True, 'msg': '更新成功'})
except Exception as e:
return jsonify({'success': False, 'error': str(e)})
@admin_bp.route('/api/admin/user/<int:uid>', methods=['DELETE'])
@admin_required
def admin_delete_user(uid):
from flask import session
if session.get('user_id') == uid:
return jsonify({'success': False, 'error': '不能删除当前登录账号'})
role, current_row = _get_current_admin_role()
if not role:
return jsonify({'success': False, 'error': '需要管理员权限'}), 403
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute("SELECT id, role, created_by_id FROM users WHERE id = %s", (uid,))
target = cur.fetchone()
if not target:
conn.close()
return jsonify({'success': False, 'error': '用户不存在'})
if target.get('role') == 'super_admin':
conn.close()
return jsonify({'success': False, 'error': '不能删除超级管理员'})
if role == 'admin':
if target.get('role') != 'normal' or target.get('created_by_id') != current_row['id']:
conn.close()
return jsonify({'success': False, 'error': '只能删除自己创建的普通用户'}), 403
cur.execute("DELETE FROM users WHERE id = %s", (uid,))
affected = cur.rowcount
conn.commit()
conn.close()
if affected == 0:
return jsonify({'success': False, 'error': '用户不存在'})
return jsonify({'success': True, 'msg': '删除成功'})
except Exception as e:
return jsonify({'success': False, 'error': str(e)})
@admin_bp.route('/api/admin/user/<int:uid>/column-permissions')
@login_required
def admin_user_column_permissions(uid):
"""获取指定用户的栏目权限列表:当前用户只能查自己,管理员可查任意用户。超级管理员返回全部栏目。"""
current_uid = session.get('user_id')
menu_type = (request.args.get('menu_type') or '').strip().lower()
if current_uid != uid:
role, _ = _get_current_admin_role()
if not role:
return jsonify({'success': False, 'error': '无权查看该用户的栏目权限'}), 403
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute("SELECT role FROM users WHERE id = %s", (uid,))
user_row = cur.fetchone()
valid_menu_type = menu_type if menu_type in ('app', 'admin') else ''
if user_row and (user_row.get('role') or '').strip() == 'super_admin':
sql = """
SELECT id, name, column_key, menu_type, route_path, sort_order, created_at
FROM columns
"""
params = []
if valid_menu_type:
sql += " WHERE menu_type = %s"
params.append(valid_menu_type)
sql += " ORDER BY sort_order ASC, id ASC"
cur.execute(sql, tuple(params))
rows = cur.fetchall()
else:
sql = """
SELECT c.id, c.name, c.column_key, c.menu_type, c.route_path, c.sort_order, c.created_at
FROM columns c
INNER JOIN user_column_permission ucp ON ucp.column_id = c.id
WHERE ucp.user_id = %s
"""
params = [uid]
if valid_menu_type:
sql += " AND c.menu_type = %s"
params.append(valid_menu_type)
sql += " ORDER BY c.sort_order ASC, c.id ASC"
cur.execute(sql, tuple(params))
rows = cur.fetchall()
conn.close()
items = [
{
'id': r['id'],
'name': r['name'],
'column_key': r['column_key'],
'menu_type': r.get('menu_type') or 'app',
'route_path': r.get('route_path') or '',
'sort_order': r.get('sort_order') or 0,
'created_at': r['created_at'].strftime('%Y-%m-%d %H:%M') if r.get('created_at') else '',
}
for r in rows
]
return jsonify({'success': True, 'items': items})
except Exception as e:
return jsonify({'success': False, 'error': str(e)})
@admin_bp.route('/api/admin/history')
@admin_required
def admin_history():
"""管理员分页获取所有生成记录,支持按用户和时间筛选"""
page = max(1, int(request.args.get('page', 1)))
page_size = min(50, max(10, int(request.args.get('page_size', 15))))
offset = (page - 1) * page_size
user_id = request.args.get('user_id', type=int)
time_start = (request.args.get('time_start') or '').strip()
time_end = (request.args.get('time_end') or '').strip()
conditions, params = [], []
if user_id:
conditions.append("h.user_id = %s")
params.append(user_id)
if time_start:
conditions.append("h.created_at >= %s")
params.append(time_start)
if time_end:
conditions.append("h.created_at <= %s")
params.append(time_end + ' 23:59:59' if len(time_end) <= 10 else time_end)
where_clause = " AND ".join(conditions) if conditions else "1=1"
params_count = params[:]
params.extend([page_size, offset])
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute(
"""SELECT h.id, h.user_id, h.created_at, h.panel_type, h.original_urls, h.params, h.result_urls,
h.long_image_url, u.username
FROM image_history h
LEFT JOIN users u ON h.user_id = u.id
WHERE """ + where_clause + """ ORDER BY h.created_at DESC LIMIT %s OFFSET %s""",
params,
)
rows = cur.fetchall()
cur.execute("SELECT COUNT(*) as total FROM image_history h WHERE " + where_clause, params_count)
total = cur.fetchone()['total']
conn.close()
items = []
for r in rows:
items.append({
'id': r['id'],
'user_id': r['user_id'],
'username': r.get('username') or '-',
'created_at': r['created_at'].strftime('%Y-%m-%d %H:%M') if r['created_at'] else '',
'panel_type': r['panel_type'] or '',
'original_urls': _parse_json(r['original_urls'], []),
'params': _parse_json(r['params'], {}),
'result_urls': _parse_json(r['result_urls'], []),
'long_image_url': (r.get('long_image_url') or '').strip() or None,
})
return jsonify({'success': True, 'items': items, 'total': total, 'page': page, 'page_size': page_size})
except Exception as e:
return jsonify({'success': False, 'error': str(e)})

View File

@@ -1,107 +0,0 @@
"""
认证蓝图:登录、登出、登录状态校验
"""
from flask import Blueprint, request, redirect, url_for, session, jsonify
from werkzeug.security import check_password_hash
from app_common import (
get_db,
_render_html,
_is_session_user_valid,
login_required,
BASE_DIR,
)
from tool.devices import DeviceIDGenerator
auth_bp = Blueprint('auth', __name__)
@auth_bp.route('/login', methods=['GET', 'POST'])
def login():
if session.get('user_id') and _is_session_user_valid():
return redirect(url_for('main.home'))
if request.method == 'POST':
data = request.get_json() if request.is_json else request.form
username = (data.get('username') or '').strip()
password = data.get('password') or ''
if not username or not password:
if request.is_json:
return jsonify({'success': False, 'error': '请输入用户名和密码'})
return _render_html('login.html', error='请输入用户名和密码')
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute(
"SELECT id, password_hash, machine, is_admin FROM users WHERE username = %s",
(username,)
)
row = cur.fetchone()
if row and check_password_hash(row['password_hash'], password):
current_machine = DeviceIDGenerator().get_device_id()
stored_machine = (row.get('machine') or '').strip()
if not stored_machine:
with conn.cursor() as cur:
cur.execute("UPDATE users SET machine = %s WHERE id = %s", (current_machine, row['id']))
conn.commit()
conn.close()
session.permanent = True
session['user_id'] = row['id']
session['username'] = username
if request.is_json:
return jsonify({'success': True, 'redirect': url_for('main.home')})
return redirect(url_for('main.home'))
print("验证设备",stored_machine)
print("当前设备",current_machine)
if stored_machine != current_machine and row.get("is_admin") != 1:
conn.close()
err_msg = '当前设备与首次登录设备不一致,请在原设备上登录'
if request.is_json:
return jsonify({'success': False, 'error': err_msg})
return _render_html('login.html', error=err_msg)
conn.close()
session.permanent = True
session['user_id'] = row['id']
session['username'] = username
if request.is_json:
return jsonify({'success': True, 'redirect': url_for('main.home')})
return redirect(url_for('main.home'))
conn.close()
except Exception as e:
if request.is_json:
return jsonify({'success': False, 'error': str(e)})
return _render_html('login.html', error='登录失败,请稍后重试')
if request.is_json:
return jsonify({'success': False, 'error': '用户名或密码错误'})
return _render_html('login.html', error='用户名或密码错误')
return _render_html('login.html')
@auth_bp.route('/api/auth/check')
@login_required
def api_auth_check():
"""校验登录状态,用于页面加载时判断是否已登录;同时校验机器码是否与首次登录设备一致"""
if not session.get('user_id'):
return jsonify({'logged_in': False})
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute("SELECT machine, is_admin FROM users WHERE id = %s", (session['user_id'],))
row = cur.fetchone()
conn.close()
if not row:
return jsonify({'logged_in': False})
stored_machine = (row.get('machine') or '').strip()
if stored_machine:
current_machine = DeviceIDGenerator().get_device_id()
if stored_machine != current_machine and row.get("is_admin") != 1:
session.clear()
return jsonify({'logged_in': False, 'error': '当前设备与首次登录设备不一致'})
except Exception:
return jsonify({'logged_in': False})
return jsonify({'logged_in': True, 'redirect': url_for('main.home')})
@auth_bp.route('/logout')
def logout():
session.clear()
return redirect(url_for('auth.login'))

View File

@@ -1,728 +0,0 @@
"""
品牌爬虫蓝图:展开文件夹、运行任务、任务列表/详情、下载结果
"""
import os
import json
import threading
import traceback
import zipfile
import io
import time
import requests
from queue import Queue, Empty
from concurrent.futures import ThreadPoolExecutor, as_completed
from urllib.parse import urlparse
from flask import Blueprint, request, jsonify, session, send_file, redirect, Response
from app_common import get_db, login_required, BASE_DIR
from config import bucket_path,JAVA_API_BASE
from brand_spider.main import single_file_handle, TaskCancelledError
brand_bp = Blueprint('brand', __name__)
BRAND_OUTPUT_DIR = os.path.join(BASE_DIR, 'brand_output')
OSS_PREFIX = "brand_results"
# 任务完成时推送给前端的 SSE 队列task_id -> [Queue, ...]
_task_event_queues = {}
_task_event_lock = threading.Lock()
# 任务行级进度(当前处理文件的行数/总行数),仅存内存,不落库:
# task_id -> {
# 'file_index': int, # 当前处理第几个文件
# 'file_total': int, # 总文件数
# 'file_path': str, # 当前文件路径
# 'current_line': int, # 当前行号(或当前品牌序号)
# 'total_lines': int, # 总行数(或总品牌数)
# }
_task_line_progress = {}
_task_line_progress_lock = threading.Lock()
_cancelled_brand_tasks = set()
_cancelled_brand_tasks_lock = threading.Lock()
def _mark_task_cancelled(task_id):
with _cancelled_brand_tasks_lock:
_cancelled_brand_tasks.add(task_id)
def _is_task_cancelled(task_id):
with _cancelled_brand_tasks_lock:
return task_id in _cancelled_brand_tasks
def _get_uid_from_request_headers():
"""
从请求头读取 uid。
前端会在 headers 里携带 uid用于确定本次请求的归属用户。
"""
uid = request.headers.get('uid')
if uid is None:
return None
uid = str(uid).strip()
if not uid:
return None
try:
return int(uid)
except Exception:
return None
def _resolve_user_id():
"""
优先使用请求头 uid没有请求头 uid 时回退到 session['user_id']。
若二者同时存在但不一致,则视为无效请求并返回 None。
"""
req_uid = _get_uid_from_request_headers()
if req_uid is not None:
sess_uid = session.get('user_id')
if sess_uid is not None:
try:
if int(sess_uid) != int(req_uid):
return None
except Exception:
if str(sess_uid) != str(req_uid):
return None
return req_uid
return session.get('user_id')
def _get_user_id_or_error():
user_id = _resolve_user_id()
if not user_id:
return None, (jsonify({'success': False, 'error': '未登录或缺少uid'}), 401)
try:
return int(user_id), None
except Exception:
return None, (jsonify({'success': False, 'error': 'uid无效'}), 400)
def _push_task_event(task_id, event):
"""向订阅了该任务的所有 SSE 连接推送事件,并移除该任务的队列列表。
同时清理内存中的行级进度,不通过数据库中转行级进度。"""
with _task_event_lock:
queues = _task_event_queues.pop(task_id, [])
for q in queues:
try:
q.put_nowait(event)
except Exception:
pass
# 任务进入终态时,顺便清理行级进度缓存
with _task_line_progress_lock:
_task_line_progress.pop(task_id, None)
status = (event or {}).get('status') if isinstance(event, dict) else None
if status in ('success', 'failed', 'cancelled'):
with _cancelled_brand_tasks_lock:
_cancelled_brand_tasks.discard(task_id)
def _expand_folder_xlsx(folder_path):
"""返回文件夹下所有 .xlsx 文件的绝对路径列表"""
if not folder_path or not os.path.isdir(folder_path):
return []
paths = []
for name in os.listdir(folder_path):
if name.endswith('.xlsx') or name.endswith('.XLSX'):
paths.append(os.path.normpath(os.path.join(folder_path, name)))
return paths
def _upload_local_xlsx_to_oss(local_path, user_id):
"""将本地 xlsx 文件上传到 OSS返回 (url, None) 或 (None, error_message)。"""
if not local_path or not os.path.isfile(local_path):
return None, "文件不存在"
try:
from ali_oss import upload_file as oss_upload_file
except ImportError:
return None, "OSS 模块未配置"
base = os.path.basename(local_path)
safe_base = "".join(c if c.isalnum() or c in '-_.' else '_' for c in base)
if not safe_base.endswith('.xlsx'):
safe_base = safe_base + '.xlsx'
key = f"{bucket_path}brand_input/{user_id}/{int(time.time() * 1000)}/{safe_base}"
try:
with open(local_path, "rb") as f:
content = f.read()
url = oss_upload_file(content, key)
return url, None
except Exception as e:
return None, str(e)
def _run_brand_single(taskid,brand_ls, fileUrl,strategy,totalLines,chunkTotal,chunkIndex):
""""""
invalidBrands = [] # 不符合品牌的数据
queryFailedBrands = [] # 查询失败的数据
keptRows = []
for brand in brand_ls:
if _is_task_cancelled(taskid):
raise TaskCancelledError("任务已取消")
faild_data,query_faild_data = single_file_handle(brand,strategy)
invalidBrands.extend(faild_data)
queryFailedBrands.extend(query_faild_data)
if len(faild_data) == 0 and len(query_faild_data) == 0:
keptRows.append(brand)
data = { "strategy": strategy ,
"files": [
{
"fileUrl": fileUrl,
"originalFilename": "",
"relativePath": "",
"mainSheetName": "",
"chunkIndex": chunkIndex,
"chunkTotal": chunkTotal,
"totalLines": totalLines,
"keptRows": keptRows,
"invalidBrands": invalidBrands,
"queryFailedBrands": queryFailedBrands
}
]
}
resp = requests.post(f"{JAVA_API_BASE}/api/brand/tasks/{taskid}/result",headers={"accept":"application/json"},
# json={ "strategy": strategy ,
# "files": [
# { "fileUrl": fileUrl,"originalFilename": "","relativePath": "",
# "mainSheetName": "","columns": [],"keptRows": [],
# "invalidBrands": invalidBrands,"queryFailedBrands": queryFailedBrands
# }]})
json=data)
# print(data)
# print(taskid,brand_ls, fileUrl)
print("提交结果",data,"\n-->",resp.text)
return True
def _background_brand_task(task_id,data):
"""
后台执行爬虫任务
参数示例:
data : {
"taskId": 348,
"strategy": "Terms",
"files": [
{
"fileIndex": 1,
"fileUrl": "ab8cce4878754bfd89b4cd3ed20e1395",
"originalFilename": "品牌样例.xlsx",
"relativePath": "店铺A/品牌样例.xlsx",
"sheetName": "Sheet1",
"columns": [
"品牌",
"ASIH",
"状态",
"时间"
],
"rows": [
{
"品牌": "YQAUTEC",
"ASIH": "B0F28NZ752",
"状态": "",
"时间": "",
"__rowIndex": 2
}
],
"uniqueBrands": [
"YQAUTEC"
]
}
]
}
"""
try:
file_ls = data.get("files")
strategy = data.get("strategy")
if not isinstance(file_ls, list):
file_ls = []
tasks = []
for file_data in file_ls:
rows = file_data.get("rows") or []
brand_ls = [i.get("品牌") for i in rows if i.get("品牌")]
fileUrl = file_data.get("fileUrl")
if not brand_ls:
continue
file_chunks = [brand_ls[i:i + 5] for i in range(0, len(brand_ls), 5)]
chunk_total = len(file_chunks)
total_lines = len(brand_ls)
for chunk_index, chunk in enumerate(file_chunks, start=1):
tasks.append((chunk, fileUrl, total_lines, chunk_total, chunk_index))
if tasks:
max_workers = min(8, len(tasks))
with ThreadPoolExecutor(max_workers=max_workers) as executor:
if _is_task_cancelled(task_id):
_push_task_event(task_id, {'status': 'cancelled'})
return
futures = [
executor.submit(_run_brand_single, task_id, chunk, file_url, strategy, total_lines, chunk_total, chunk_index)
for chunk, file_url, total_lines, chunk_total, chunk_index in tasks
]
for future in as_completed(futures):
if _is_task_cancelled(task_id):
for f in futures:
f.cancel()
_push_task_event(task_id, {'status': 'cancelled'})
return
future.result()
_push_task_event(task_id, {'status': 'success'})
except Exception as e:
print("执行出错",traceback.format_exc())
_push_task_event(task_id, {'status': 'failed', 'error_message': str(e)})
print("执行完成")
def _parse_json(val, default=None):
if val is None:
return default if default is not None else []
if isinstance(val, (list, dict)):
return val
try:
return json.loads(val)
except Exception:
return default if default is not None else []
def _is_url(s):
if not isinstance(s, str) or not s.strip():
return False
return s.strip().startswith('http://') or s.strip().startswith('https://')
def _normalize_result_paths(raw):
"""
将 result_paths 规范化为 (url_list, zip_url)。
支持旧格式 list 或新格式 dict {"urls": [...], "zip_url": "..."}。
"""
if not raw:
return [], None
if isinstance(raw, dict):
urls = raw.get('urls') or []
if not isinstance(urls, list):
urls = []
zip_url = raw.get('zip_url')
if zip_url and not isinstance(zip_url, str):
zip_url = None
return urls, zip_url
if isinstance(raw, list):
return raw, None
return [], None
@brand_bp.route('/api/brand/expand-folder', methods=['POST'])
@login_required
def api_brand_expand_folder():
"""展开文件夹,返回其下所有 .xlsx 文件路径列表"""
try:
_, err = _get_user_id_or_error()
if err:
return err
data = request.get_json() or {}
folder = (data.get('folder') or '').strip()
if not folder:
return jsonify({'success': False, 'error': '请提供 folder 路径'}), 400
paths = _expand_folder_xlsx(folder)
return jsonify({'success': True, 'paths': paths})
except Exception as e:
return jsonify({'success': False, 'error': str(e)}), 500
def _expand_folder_xlsx_recursive_items(folder_path):
"""返回文件夹下所有 .xlsx 文件的绝对路径和相对路径列表(递归)"""
if not folder_path or not os.path.isdir(folder_path):
return []
items = []
root = os.path.normpath(folder_path)
for current_root, _, filenames in os.walk(root):
for name in filenames:
if not (name.endswith('.xlsx') or name.endswith('.XLSX')):
continue
absolute_path = os.path.normpath(os.path.join(current_root, name))
relative_path = os.path.relpath(absolute_path, root).replace('\\', '/')
items.append({
'absolutePath': absolute_path,
'relativePath': relative_path,
})
items.sort(key=lambda item: item['relativePath'])
return items
@brand_bp.route('/api/brand/expand-folder-recursive', methods=['POST'])
@login_required
def api_brand_expand_folder_recursive():
"""递归展开文件夹,返回 .xlsx 文件绝对路径及相对路径列表"""
try:
_, err = _get_user_id_or_error()
if err:
return err
data = request.get_json() or {}
folder = (data.get('folder') or '').strip()
if not folder:
return jsonify({'success': False, 'error': '请提供 folder 路径'}), 400
items = _expand_folder_xlsx_recursive_items(folder)
return jsonify({'success': True, 'items': items})
except Exception as e:
return jsonify({'success': False, 'error': str(e)}), 500
@brand_bp.route('/api/brand/run', methods=['POST'])
# @login_required
def api_brand_run():
"""立即运行:创建任务并后台执行,立即返回 task_id前端可轮询进度与取消"""
try:
data = request.get_json() or {}
paths = data.get('paths') or []
# task_type: 1=立即执行2=添加任务;此接口默认 1
try:
task_type = int(data.get('task_type') or 1)
except Exception:
task_type = 1
if task_type not in (1, 2):
task_type = 1
strategy = (data.get('strategy') or 'Terms').strip()
if strategy not in ('Terms', 'Simple'):
strategy = 'Terms'
if not isinstance(paths, list):
paths = []
paths = [p.strip() for p in paths if p and isinstance(p, str) and os.path.isfile(p.strip())]
if not paths:
return jsonify({'success': False, 'error': '没有有效的 xlsx 文件路径'}), 400
# 先上传到 OSS获取链接再入库
user_id, err = _get_user_id_or_error()
if err:
return err
urls = []
for p in paths:
url, err = _upload_local_xlsx_to_oss(p, user_id)
if err:
return jsonify({'success': False, 'error': f'上传文件失败: {os.path.basename(p)} - {err}'}), 500
base_name = os.path.basename(p)
urls.append({"fileUrl": url,"originalFilename": base_name,"relativePath": p })
# 请求提交
resp = requests.post(f"{JAVA_API_BASE}/api/brand/tasks?userId={user_id}",headers={
"content-type":"application/json",
},data=json.dumps({ "files": urls, "strategy": strategy,"taskType": 1,"archiveName": ""}))
print({ "files": urls, "strategy": strategy,"taskType": 1,"archiveName": ""})
print(f"{JAVA_API_BASE}/api/brand/tasks?userId={user_id}")
resp_data = resp.json()
# print(f"{JAVA_API_BASE}/api/brand/tasks?userId={user_id} 返回:",resp_data,"状态:",resp.status_code)
if resp_data.get("success"):
task_id = resp_data.get("data").get("taskId")
data = resp_data.get("data")
threading.Thread(target=_background_brand_task, args=(task_id,data), daemon=True).start()
return jsonify({'success': True, 'task_id': task_id})
else:
print(resp_data)
return jsonify({'success': False, 'error': "请求接口失败"}), 500
except Exception as e:
traceback.print_exc()
return jsonify({'success': False, 'error': str(e)}), 500
@brand_bp.route('/api/brand/tasks', methods=['GET', 'POST'])
# @login_required
def api_brand_tasks():
"""GET: 获取当前用户的任务列表POST: 添加任务(后台执行)"""
if request.method == 'GET':
try:
user_id, err = _get_user_id_or_error()
if err:
return err
resp = requests.get(f"{JAVA_API_BASE}/api/brand/tasks", params={
"userId": user_id
})
resp_data = resp.json()
print("获取列表",resp.text)
if resp_data.get("success"):
items = resp_data.get("data").get("items")
return jsonify({'success': True, 'items': items})
return jsonify({'success': True, 'items': []})
except Exception as e:
traceback.print_exc()
print("获取列表失败",e)
return jsonify({'success': False, 'error': str(e)}), 500
# POST
try:
data = request.get_json() or {}
paths = data.get('paths') or []
# task_type: 1=立即执行2=添加任务;此接口默认 2
try:
task_type = int(data.get('task_type') or 2)
except Exception:
task_type = 2
if task_type not in (1, 2):
task_type = 2
strategy = (data.get('strategy') or 'Terms').strip()
if strategy not in ('Terms', 'Simple'):
strategy = 'Terms'
if not isinstance(paths, list):
paths = []
paths = [p.strip() for p in paths if p and isinstance(p, str)]
if not paths:
return jsonify({'success': False, 'error': '请提供至少一个文件路径或链接'}), 400
user_id, err = _get_user_id_or_error()
if err:
return err
urls = []
for p in paths:
url, err = _upload_local_xlsx_to_oss(p, user_id)
if err:
return jsonify({'success': False, 'error': f'上传文件失败: {os.path.basename(p)} - {err}'}), 500
base_name = os.path.basename(p)
urls.append({"fileUrl": url, "originalFilename": base_name, "relativePath": p})
params = {
"userId": user_id
}
# resp = requests.post(f"{JAVA_API_BASE}/api/brand/tasks?userId={user_id}",headers={
# "content-type":"application/json",
# },data=json.dumps({ "files": urls, "strategy": strategy,"taskType": 1,"archiveName": ""}))
req_data = {"files": urls, "strategy": strategy, "taskType": 2, "archiveName": ""}
resp = requests.post(f"{JAVA_API_BASE}/api/brand/tasks?userId={user_id}", headers={"content-type": "application/json"},
data=json.dumps(req_data))
print(req_data)
# print(f"{JAVA_API_BASE}/api/brand/tasks?userId={user_id},返回", resp.text)
resp_data = resp.json()
if resp_data.get("success"):
task_id = resp_data.get("data").get("taskId")
data = resp_data.get("data")
return jsonify({'success': True, 'task_id': task_id})
return jsonify({'success': False, 'task_id': "请求异常"})
except Exception as e:
return jsonify({'success': False, 'error': str(e)}), 500
@brand_bp.route('/api/brand/tasks/<int:task_id>')
# @login_required
def api_brand_task_detail(task_id):
"""获取单个任务详情(用于轮询状态)"""
try:
resp = requests.get(f"{JAVA_API_BASE}/api/brand/tasks/{task_id}")
resp_data = resp.json()
if resp_data.get("success"):
task = resp_data["data"]["task"]
return jsonify({
'success': True,
'task': task
})
except Exception as e:
return jsonify({'success': False, 'error': str(e)}), 500
@brand_bp.route('/api/brand/tasks/<int:task_id>/line-progress')
# @login_required
def api_brand_task_line_progress(task_id):
"""
获取任务的“当前文件行级进度”(当前行数/总行数)。
该信息仅存放在内存字典 _task_line_progress 中,不写入数据库。
"""
try:
resp = requests.get(f"{JAVA_API_BASE}/api/brand/tasks/{task_id}")
resp_data = resp.json()
if resp_data.get("success"):
if resp_data["data"]["line_progress"]["has_progress"]:
info = resp_data["data"]["line_progress"]["info"]
resp = {
'file_index': int(info.get('file_index') or 0),
'file_total': int(info.get('file_total') or 0),
'file_name': info.get("file_name",""),
'current_line': int(info.get('current_line') or 0),
'total_lines': int(info.get('total_lines') or 0),
}
return jsonify({'success': True, 'has_progress': True, 'info': resp})
raise RuntimeError("获取进度失败")
except Exception as e:
return jsonify({'success': False, 'error': str(e)}), 500
@brand_bp.route('/api/brand/tasks/<int:task_id>/events')
# @login_required
def api_brand_task_events(task_id):
"""SSE任务完成时后端主动推送事件前端监听后隐藏进度条与取消按钮"""
user_id, err = _get_user_id_or_error()
if err:
return err
def _task_status_event():
conn = get_db()
try:
with conn.cursor() as cur:
cur.execute(
"SELECT status, error_message FROM brand_crawl_tasks WHERE id = %s AND user_id = %s",
(task_id, user_id)
)
row = cur.fetchone()
finally:
conn.close()
if not row:
return None
st = (row.get('status') or '').lower()
if st in ('success', 'failed', 'cancelled'):
return {'status': st, 'error_message': (row.get('error_message') or '').strip() or None}
return None
# 若任务已处于终态,直接返回一条事件后结束
ev = _task_status_event()
if ev is not None:
def _one_shot():
yield "data: " + json.dumps(ev, ensure_ascii=False) + "\n\n"
return Response(
_one_shot(),
mimetype='text/event-stream',
headers={'Cache-Control': 'no-cache', 'X-Accel-Buffering': 'no'}
)
# 否则注册队列,等待后台任务完成时推送
q = Queue()
with _task_event_lock:
_task_event_queues.setdefault(task_id, []).append(q)
def _stream():
try:
while True:
try:
event = q.get(timeout=20)
yield "data: " + json.dumps(event, ensure_ascii=False) + "\n\n"
return
except Empty:
yield ": keepalive\n\n"
finally:
with _task_event_lock:
lst = _task_event_queues.get(task_id, [])
if q in lst:
lst.remove(q)
if not lst:
_task_event_queues.pop(task_id, None)
return Response(
_stream(),
mimetype='text/event-stream',
headers={'Cache-Control': 'no-cache', 'X-Accel-Buffering': 'no'}
)
@brand_bp.route('/api/brand/tasks/<int:task_id>/cancel', methods=['POST'])
# @login_required
def api_brand_task_cancel(task_id):
"""取消正在执行或等待中的任务pending/running 均可取消)"""
try:
user_id, err = _get_user_id_or_error()
if err:
return err
resp = requests.post(
f"{JAVA_API_BASE}/api/brand/tasks/{task_id}/cancel",
headers={"accept": "application/json"},
timeout=20
)
try:
resp_data = resp.json()
except Exception:
return jsonify({'success': False, 'error': '取消接口返回非 JSON'}), 500
if not resp_data.get('success'):
return jsonify({'success': False, 'error': '第三方取消任务失败'}), 400
_mark_task_cancelled(task_id)
_push_task_event(task_id, {'status': 'cancelled'})
return jsonify({'success': True})
except Exception as e:
return jsonify({'success': False, 'error': str(e)}), 500
@brand_bp.route('/api/brand/tasks/<int:task_id>', methods=['DELETE'])
@login_required
def api_brand_task_delete(task_id):
"""删除任务(仅限非 running 状态)"""
try:
user_id, err = _get_user_id_or_error()
if err:
return err
resp = requests.delete(
f"{JAVA_API_BASE}/api/brand/tasks/{task_id}",
headers={"accept": "application/json"},
timeout=20
)
try:
resp_data = resp.json()
except Exception:
return jsonify({'success': False, 'error': '取消接口返回非 JSON'}), 500
if not resp_data.get('success'):
return jsonify({'success': False, 'error': '第三方取消任务失败'}), 400
return jsonify({'success': True})
except Exception as e:
return jsonify({'success': False, 'error': str(e)}), 500
@brand_bp.route('/api/brand/download/<int:task_id>')
@login_required
def api_brand_download(task_id):
"""下载任务结果:优先使用 OSS zip_url 重定向;否则本地/单链接/多链接按原逻辑处理"""
try:
user_id, err = _get_user_id_or_error()
if err:
return err
conn = get_db()
with conn.cursor() as cur:
cur.execute(
"SELECT result_paths FROM brand_crawl_tasks WHERE id = %s AND user_id = %s",
(task_id, user_id)
)
row = cur.fetchone()
conn.close()
if not row or not row.get('result_paths'):
return jsonify({'success': False, 'error': '无结果可下载'}), 404
raw = row['result_paths']
if isinstance(raw, str):
raw = _parse_json(raw)
url_list, zip_url = _normalize_result_paths(raw)
# 新格式:有 zip_url 直接重定向到 OSS
if zip_url and _is_url(zip_url):
return redirect(zip_url, code=302)
# 兼容旧格式result_paths 可能为 list本地路径或 url
if not url_list and isinstance(raw, list):
url_list = raw
local_paths = [p for p in url_list if isinstance(p, str) and not _is_url(p) and os.path.isfile(p)]
url_paths = [p.strip() for p in url_list if isinstance(p, str) and _is_url(p)]
if len(url_paths) == 1 and not local_paths:
return redirect(url_paths[0], code=302)
if len(local_paths) == 1 and not url_paths:
return send_file(
local_paths[0],
as_attachment=True,
download_name=os.path.basename(local_paths[0])
)
if local_paths or url_paths:
buf = io.BytesIO()
with zipfile.ZipFile(buf, 'w', zipfile.ZIP_DEFLATED) as zf:
for p in local_paths:
zf.write(p, os.path.basename(p))
for url in url_paths:
try:
import requests as req
r = req.get(url, timeout=30, stream=True)
r.raise_for_status()
name = os.path.basename(urlparse(url).path) or ('file_%s' % (url_paths.index(url)))
if not name or name == 'file_%s' % url_paths.index(url):
name = 'download_%s' % url_paths.index(url)
zf.writestr(name, r.content)
except Exception:
pass
buf.seek(0)
return send_file(
buf,
mimetype='application/zip',
as_attachment=True,
download_name='brand_task_%s.zip' % task_id
)
return jsonify({'success': False, 'error': '结果文件不存在或链接不可用'}), 404
except Exception as e:
return jsonify({'success': False, 'error': str(e)}), 500

View File

@@ -1,726 +0,0 @@
"""
品牌爬虫蓝图:展开文件夹、运行任务、任务列表/详情、下载结果
"""
import os
import json
import threading
import traceback
import zipfile
import io
import time
import requests
from queue import Queue, Empty
from concurrent.futures import ThreadPoolExecutor, as_completed
from urllib.parse import urlparse
from flask import Blueprint, request, jsonify, session, send_file, redirect, Response
from app_common import get_db, login_required, BASE_DIR
from config import bucket_path,JAVA_API_BASE
from brand_spider.main import single_file_handle, TaskCancelledError
brand_bp = Blueprint('brand', __name__)
BRAND_OUTPUT_DIR = os.path.join(BASE_DIR, 'brand_output')
OSS_PREFIX = "brand_results"
# 任务完成时推送给前端的 SSE 队列task_id -> [Queue, ...]
_task_event_queues = {}
_task_event_lock = threading.Lock()
# 任务行级进度(当前处理文件的行数/总行数),仅存内存,不落库:
# task_id -> {
# 'file_index': int, # 当前处理第几个文件
# 'file_total': int, # 总文件数
# 'file_path': str, # 当前文件路径
# 'current_line': int, # 当前行号(或当前品牌序号)
# 'total_lines': int, # 总行数(或总品牌数)
# }
_task_line_progress = {}
_task_line_progress_lock = threading.Lock()
_cancelled_brand_tasks = set()
_cancelled_brand_tasks_lock = threading.Lock()
def _mark_task_cancelled(task_id):
with _cancelled_brand_tasks_lock:
_cancelled_brand_tasks.add(task_id)
def _is_task_cancelled(task_id):
with _cancelled_brand_tasks_lock:
return task_id in _cancelled_brand_tasks
def _get_uid_from_request_headers():
"""
从请求头读取 uid。
前端会在 headers 里携带 uid用于确定本次请求的归属用户。
"""
uid = request.headers.get('uid')
if uid is None:
return None
uid = str(uid).strip()
if not uid:
return None
try:
return int(uid)
except Exception:
return None
def _resolve_user_id():
"""
优先使用请求头 uid没有请求头 uid 时回退到 session['user_id']。
若二者同时存在但不一致,则视为无效请求并返回 None。
"""
req_uid = _get_uid_from_request_headers()
if req_uid is not None:
sess_uid = session.get('user_id')
if sess_uid is not None:
try:
if int(sess_uid) != int(req_uid):
return None
except Exception:
if str(sess_uid) != str(req_uid):
return None
return req_uid
return session.get('user_id')
def _get_user_id_or_error():
user_id = _resolve_user_id()
if not user_id:
return None, (jsonify({'success': False, 'error': '未登录或缺少uid'}), 401)
try:
return int(user_id), None
except Exception:
return None, (jsonify({'success': False, 'error': 'uid无效'}), 400)
def _push_task_event(task_id, event):
"""向订阅了该任务的所有 SSE 连接推送事件,并移除该任务的队列列表。
同时清理内存中的行级进度,不通过数据库中转行级进度。"""
with _task_event_lock:
queues = _task_event_queues.pop(task_id, [])
for q in queues:
try:
q.put_nowait(event)
except Exception:
pass
# 任务进入终态时,顺便清理行级进度缓存
with _task_line_progress_lock:
_task_line_progress.pop(task_id, None)
status = (event or {}).get('status') if isinstance(event, dict) else None
if status in ('success', 'failed', 'cancelled'):
with _cancelled_brand_tasks_lock:
_cancelled_brand_tasks.discard(task_id)
def _expand_folder_xlsx(folder_path):
"""返回文件夹下所有 .xlsx 文件的绝对路径列表"""
if not folder_path or not os.path.isdir(folder_path):
return []
paths = []
for name in os.listdir(folder_path):
if name.endswith('.xlsx') or name.endswith('.XLSX'):
paths.append(os.path.normpath(os.path.join(folder_path, name)))
return paths
def _upload_local_xlsx_to_oss(local_path, user_id):
"""将本地 xlsx 文件上传到 OSS返回 (url, None) 或 (None, error_message)。"""
if not local_path or not os.path.isfile(local_path):
return None, "文件不存在"
try:
from ali_oss import upload_file as oss_upload_file
except ImportError:
return None, "OSS 模块未配置"
base = os.path.basename(local_path)
safe_base = "".join(c if c.isalnum() or c in '-_.' else '_' for c in base)
if not safe_base.endswith('.xlsx'):
safe_base = safe_base + '.xlsx'
key = f"{bucket_path}brand_input/{user_id}/{int(time.time() * 1000)}/{safe_base}"
try:
with open(local_path, "rb") as f:
content = f.read()
url = oss_upload_file(content, key)
return url, None
except Exception as e:
return None, str(e)
def _run_brand_single(taskid,brand_ls, fileUrl,strategy,totalLines,chunkTotal,chunkIndex):
""""""
invalidBrands = [] # 不符合品牌的数据
queryFailedBrands = [] # 查询失败的数据
keptRows = []
for brand in brand_ls:
if _is_task_cancelled(taskid):
raise TaskCancelledError("任务已取消")
faild_data,query_faild_data = single_file_handle(brand,strategy)
invalidBrands.extend(faild_data)
queryFailedBrands.extend(query_faild_data)
if len(faild_data) == 0 and len(query_faild_data) == 0:
keptRows.append(brand)
data = { "strategy": strategy ,
"files": [
{
"fileUrl": fileUrl,
"originalFilename": "",
"relativePath": "",
"mainSheetName": "",
"chunkIndex": chunkIndex,
"chunkTotal": chunkTotal,
"totalLines": totalLines,
"keptRows": keptRows,
"invalidBrands": invalidBrands,
"queryFailedBrands": queryFailedBrands
}
]
}
resp = requests.post(f"{JAVA_API_BASE}/api/brand/tasks/{taskid}/result",headers={"accept":"application/json"},
# json={ "strategy": strategy ,
# "files": [
# { "fileUrl": fileUrl,"originalFilename": "","relativePath": "",
# "mainSheetName": "","columns": [],"keptRows": [],
# "invalidBrands": invalidBrands,"queryFailedBrands": queryFailedBrands
# }]})
json=data)
# print(data)
# print(taskid,brand_ls, fileUrl)
print("提交结果",data,"\n-->",resp.text)
return True
def _background_brand_task(task_id,data):
"""
后台执行爬虫任务
参数示例:
data : {
"taskId": 348,
"strategy": "Terms",
"files": [
{
"fileIndex": 1,
"fileUrl": "ab8cce4878754bfd89b4cd3ed20e1395",
"originalFilename": "品牌样例.xlsx",
"relativePath": "店铺A/品牌样例.xlsx",
"sheetName": "Sheet1",
"columns": [
"品牌",
"ASIH",
"状态",
"时间"
],
"rows": [
{
"品牌": "YQAUTEC",
"ASIH": "B0F28NZ752",
"状态": "",
"时间": "",
"__rowIndex": 2
}
],
"uniqueBrands": [
"YQAUTEC"
]
}
]
}
"""
try:
file_ls = data.get("files")
strategy = data.get("strategy")
if not isinstance(file_ls, list):
file_ls = []
tasks = []
for file_data in file_ls:
rows = file_data.get("rows") or []
brand_ls = [i.get("品牌") for i in rows if i.get("品牌")]
fileUrl = file_data.get("fileUrl")
if not brand_ls:
continue
for i in range(0, len(brand_ls), 5):
chunk = brand_ls[i:i + 5]
tasks.append((chunk, fileUrl))
if tasks:
max_workers = min(8, len(tasks))
with ThreadPoolExecutor(max_workers=max_workers) as executor:
if _is_task_cancelled(task_id):
_push_task_event(task_id, {'status': 'cancelled'})
return
futures = [
executor.submit(_run_brand_single, task_id, chunk, file_url, strategy,len(brand_ls),len(tasks),chunkIndex+1)
for chunkIndex,(chunk, file_url) in enumerate(tasks)
]
for future in as_completed(futures):
if _is_task_cancelled(task_id):
for f in futures:
f.cancel()
_push_task_event(task_id, {'status': 'cancelled'})
return
future.result()
_push_task_event(task_id, {'status': 'success'})
except Exception as e:
print("执行出错",traceback.format_exc())
_push_task_event(task_id, {'status': 'failed', 'error_message': str(e)})
print("执行完成")
def _parse_json(val, default=None):
if val is None:
return default if default is not None else []
if isinstance(val, (list, dict)):
return val
try:
return json.loads(val)
except Exception:
return default if default is not None else []
def _is_url(s):
if not isinstance(s, str) or not s.strip():
return False
return s.strip().startswith('http://') or s.strip().startswith('https://')
def _normalize_result_paths(raw):
"""
将 result_paths 规范化为 (url_list, zip_url)。
支持旧格式 list 或新格式 dict {"urls": [...], "zip_url": "..."}。
"""
if not raw:
return [], None
if isinstance(raw, dict):
urls = raw.get('urls') or []
if not isinstance(urls, list):
urls = []
zip_url = raw.get('zip_url')
if zip_url and not isinstance(zip_url, str):
zip_url = None
return urls, zip_url
if isinstance(raw, list):
return raw, None
return [], None
@brand_bp.route('/api/brand/expand-folder', methods=['POST'])
@login_required
def api_brand_expand_folder():
"""展开文件夹,返回其下所有 .xlsx 文件路径列表"""
try:
_, err = _get_user_id_or_error()
if err:
return err
data = request.get_json() or {}
folder = (data.get('folder') or '').strip()
if not folder:
return jsonify({'success': False, 'error': '请提供 folder 路径'}), 400
paths = _expand_folder_xlsx(folder)
return jsonify({'success': True, 'paths': paths})
except Exception as e:
return jsonify({'success': False, 'error': str(e)}), 500
def _expand_folder_xlsx_recursive_items(folder_path):
"""返回文件夹下所有 .xlsx 文件的绝对路径和相对路径列表(递归)"""
if not folder_path or not os.path.isdir(folder_path):
return []
items = []
root = os.path.normpath(folder_path)
for current_root, _, filenames in os.walk(root):
for name in filenames:
if not (name.endswith('.xlsx') or name.endswith('.XLSX')):
continue
absolute_path = os.path.normpath(os.path.join(current_root, name))
relative_path = os.path.relpath(absolute_path, root).replace('\\', '/')
items.append({
'absolutePath': absolute_path,
'relativePath': relative_path,
})
items.sort(key=lambda item: item['relativePath'])
return items
@brand_bp.route('/api/brand/expand-folder-recursive', methods=['POST'])
@login_required
def api_brand_expand_folder_recursive():
"""递归展开文件夹,返回 .xlsx 文件绝对路径及相对路径列表"""
try:
_, err = _get_user_id_or_error()
if err:
return err
data = request.get_json() or {}
folder = (data.get('folder') or '').strip()
if not folder:
return jsonify({'success': False, 'error': '请提供 folder 路径'}), 400
items = _expand_folder_xlsx_recursive_items(folder)
return jsonify({'success': True, 'items': items})
except Exception as e:
return jsonify({'success': False, 'error': str(e)}), 500
@brand_bp.route('/api/brand/run', methods=['POST'])
# @login_required
def api_brand_run():
"""立即运行:创建任务并后台执行,立即返回 task_id前端可轮询进度与取消"""
try:
data = request.get_json() or {}
paths = data.get('paths') or []
# task_type: 1=立即执行2=添加任务;此接口默认 1
try:
task_type = int(data.get('task_type') or 1)
except Exception:
task_type = 1
if task_type not in (1, 2):
task_type = 1
strategy = (data.get('strategy') or 'Terms').strip()
if strategy not in ('Terms', 'Simple'):
strategy = 'Terms'
if not isinstance(paths, list):
paths = []
paths = [p.strip() for p in paths if p and isinstance(p, str) and os.path.isfile(p.strip())]
if not paths:
return jsonify({'success': False, 'error': '没有有效的 xlsx 文件路径'}), 400
# 先上传到 OSS获取链接再入库
user_id, err = _get_user_id_or_error()
if err:
return err
urls = []
for p in paths:
url, err = _upload_local_xlsx_to_oss(p, user_id)
if err:
return jsonify({'success': False, 'error': f'上传文件失败: {os.path.basename(p)} - {err}'}), 500
base_name = os.path.basename(p)
urls.append({"fileUrl": url,"originalFilename": base_name,"relativePath": p })
# 请求提交
resp = requests.post(f"{JAVA_API_BASE}/api/brand/tasks?userId={user_id}",headers={
"content-type":"application/json",
},data=json.dumps({ "files": urls, "strategy": strategy,"taskType": 1,"archiveName": ""}))
print({ "files": urls, "strategy": strategy,"taskType": 1,"archiveName": ""})
print(f"{JAVA_API_BASE}/api/brand/tasks?userId={user_id}")
resp_data = resp.json()
# print(f"{JAVA_API_BASE}/api/brand/tasks?userId={user_id} 返回:",resp_data,"状态:",resp.status_code)
if resp_data.get("success"):
task_id = resp_data.get("data").get("taskId")
data = resp_data.get("data")
threading.Thread(target=_background_brand_task, args=(task_id,data), daemon=True).start()
return jsonify({'success': True, 'task_id': task_id})
else:
print(resp_data)
return jsonify({'success': False, 'error': "请求接口失败"}), 500
except Exception as e:
traceback.print_exc()
return jsonify({'success': False, 'error': str(e)}), 500
@brand_bp.route('/api/brand/tasks', methods=['GET', 'POST'])
# @login_required
def api_brand_tasks():
"""GET: 获取当前用户的任务列表POST: 添加任务(后台执行)"""
if request.method == 'GET':
try:
user_id, err = _get_user_id_or_error()
if err:
return err
resp = requests.get(f"{JAVA_API_BASE}/api/brand/tasks", params={
"userId": user_id
})
resp_data = resp.json()
print("获取列表",resp.text)
if resp_data.get("success"):
items = resp_data.get("data").get("items")
return jsonify({'success': True, 'items': items})
return jsonify({'success': True, 'items': []})
except Exception as e:
traceback.print_exc()
print("获取列表失败",e)
return jsonify({'success': False, 'error': str(e)}), 500
# POST
try:
data = request.get_json() or {}
paths = data.get('paths') or []
# task_type: 1=立即执行2=添加任务;此接口默认 2
try:
task_type = int(data.get('task_type') or 2)
except Exception:
task_type = 2
if task_type not in (1, 2):
task_type = 2
strategy = (data.get('strategy') or 'Terms').strip()
if strategy not in ('Terms', 'Simple'):
strategy = 'Terms'
if not isinstance(paths, list):
paths = []
paths = [p.strip() for p in paths if p and isinstance(p, str)]
if not paths:
return jsonify({'success': False, 'error': '请提供至少一个文件路径或链接'}), 400
user_id, err = _get_user_id_or_error()
if err:
return err
urls = []
for p in paths:
url, err = _upload_local_xlsx_to_oss(p, user_id)
if err:
return jsonify({'success': False, 'error': f'上传文件失败: {os.path.basename(p)} - {err}'}), 500
base_name = os.path.basename(p)
urls.append({"fileUrl": url, "originalFilename": base_name, "relativePath": p})
params = {
"userId": user_id
}
# resp = requests.post(f"{JAVA_API_BASE}/api/brand/tasks?userId={user_id}",headers={
# "content-type":"application/json",
# },data=json.dumps({ "files": urls, "strategy": strategy,"taskType": 1,"archiveName": ""}))
req_data = {"files": urls, "strategy": strategy, "taskType": 2, "archiveName": ""}
resp = requests.post(f"{JAVA_API_BASE}/api/brand/tasks?userId={user_id}", headers={"content-type": "application/json"},
data=json.dumps(req_data))
print(req_data)
# print(f"{JAVA_API_BASE}/api/brand/tasks?userId={user_id},返回", resp.text)
resp_data = resp.json()
if resp_data.get("success"):
task_id = resp_data.get("data").get("taskId")
data = resp_data.get("data")
return jsonify({'success': True, 'task_id': task_id})
return jsonify({'success': False, 'task_id': "请求异常"})
except Exception as e:
return jsonify({'success': False, 'error': str(e)}), 500
@brand_bp.route('/api/brand/tasks/<int:task_id>')
# @login_required
def api_brand_task_detail(task_id):
"""获取单个任务详情(用于轮询状态)"""
try:
resp = requests.get(f"{JAVA_API_BASE}/api/brand/tasks/{task_id}")
resp_data = resp.json()
if resp_data.get("success"):
task = resp_data["data"]["task"]
return jsonify({
'success': True,
'task': task
})
except Exception as e:
return jsonify({'success': False, 'error': str(e)}), 500
@brand_bp.route('/api/brand/tasks/<int:task_id>/line-progress')
# @login_required
def api_brand_task_line_progress(task_id):
"""
获取任务的“当前文件行级进度”(当前行数/总行数)。
该信息仅存放在内存字典 _task_line_progress 中,不写入数据库。
"""
try:
resp = requests.get(f"{JAVA_API_BASE}/api/brand/tasks/{task_id}")
resp_data = resp.json()
if resp_data.get("success"):
if resp_data["data"]["line_progress"]["has_progress"]:
info = resp_data["data"]["line_progress"]["info"]
resp = {
'file_index': int(info.get('file_index') or 0),
'file_total': int(info.get('file_total') or 0),
'file_name': info.get("file_name",""),
'current_line': int(info.get('current_line') or 0),
'total_lines': int(info.get('total_lines') or 0),
}
return jsonify({'success': True, 'has_progress': True, 'info': resp})
raise RuntimeError("获取进度失败")
except Exception as e:
return jsonify({'success': False, 'error': str(e)}), 500
@brand_bp.route('/api/brand/tasks/<int:task_id>/events')
# @login_required
def api_brand_task_events(task_id):
"""SSE任务完成时后端主动推送事件前端监听后隐藏进度条与取消按钮"""
user_id, err = _get_user_id_or_error()
if err:
return err
def _task_status_event():
conn = get_db()
try:
with conn.cursor() as cur:
cur.execute(
"SELECT status, error_message FROM brand_crawl_tasks WHERE id = %s AND user_id = %s",
(task_id, user_id)
)
row = cur.fetchone()
finally:
conn.close()
if not row:
return None
st = (row.get('status') or '').lower()
if st in ('success', 'failed', 'cancelled'):
return {'status': st, 'error_message': (row.get('error_message') or '').strip() or None}
return None
# 若任务已处于终态,直接返回一条事件后结束
ev = _task_status_event()
if ev is not None:
def _one_shot():
yield "data: " + json.dumps(ev, ensure_ascii=False) + "\n\n"
return Response(
_one_shot(),
mimetype='text/event-stream',
headers={'Cache-Control': 'no-cache', 'X-Accel-Buffering': 'no'}
)
# 否则注册队列,等待后台任务完成时推送
q = Queue()
with _task_event_lock:
_task_event_queues.setdefault(task_id, []).append(q)
def _stream():
try:
while True:
try:
event = q.get(timeout=20)
yield "data: " + json.dumps(event, ensure_ascii=False) + "\n\n"
return
except Empty:
yield ": keepalive\n\n"
finally:
with _task_event_lock:
lst = _task_event_queues.get(task_id, [])
if q in lst:
lst.remove(q)
if not lst:
_task_event_queues.pop(task_id, None)
return Response(
_stream(),
mimetype='text/event-stream',
headers={'Cache-Control': 'no-cache', 'X-Accel-Buffering': 'no'}
)
@brand_bp.route('/api/brand/tasks/<int:task_id>/cancel', methods=['POST'])
# @login_required
def api_brand_task_cancel(task_id):
"""取消正在执行或等待中的任务pending/running 均可取消)"""
try:
user_id, err = _get_user_id_or_error()
if err:
return err
resp = requests.post(
f"{JAVA_API_BASE}/api/brand/tasks/{task_id}/cancel",
headers={"accept": "application/json"},
timeout=20
)
try:
resp_data = resp.json()
except Exception:
return jsonify({'success': False, 'error': '取消接口返回非 JSON'}), 500
if not resp_data.get('success'):
return jsonify({'success': False, 'error': '第三方取消任务失败'}), 400
_mark_task_cancelled(task_id)
_push_task_event(task_id, {'status': 'cancelled'})
return jsonify({'success': True})
except Exception as e:
return jsonify({'success': False, 'error': str(e)}), 500
@brand_bp.route('/api/brand/tasks/<int:task_id>', methods=['DELETE'])
@login_required
def api_brand_task_delete(task_id):
"""删除任务(仅限非 running 状态)"""
try:
user_id, err = _get_user_id_or_error()
if err:
return err
resp = requests.delete(
f"{JAVA_API_BASE}/api/brand/tasks/{task_id}",
headers={"accept": "application/json"},
timeout=20
)
try:
resp_data = resp.json()
except Exception:
return jsonify({'success': False, 'error': '取消接口返回非 JSON'}), 500
if not resp_data.get('success'):
return jsonify({'success': False, 'error': '第三方取消任务失败'}), 400
return jsonify({'success': True})
except Exception as e:
return jsonify({'success': False, 'error': str(e)}), 500
@brand_bp.route('/api/brand/download/<int:task_id>')
@login_required
def api_brand_download(task_id):
"""下载任务结果:优先使用 OSS zip_url 重定向;否则本地/单链接/多链接按原逻辑处理"""
try:
user_id, err = _get_user_id_or_error()
if err:
return err
conn = get_db()
with conn.cursor() as cur:
cur.execute(
"SELECT result_paths FROM brand_crawl_tasks WHERE id = %s AND user_id = %s",
(task_id, user_id)
)
row = cur.fetchone()
conn.close()
if not row or not row.get('result_paths'):
return jsonify({'success': False, 'error': '无结果可下载'}), 404
raw = row['result_paths']
if isinstance(raw, str):
raw = _parse_json(raw)
url_list, zip_url = _normalize_result_paths(raw)
# 新格式:有 zip_url 直接重定向到 OSS
if zip_url and _is_url(zip_url):
return redirect(zip_url, code=302)
# 兼容旧格式result_paths 可能为 list本地路径或 url
if not url_list and isinstance(raw, list):
url_list = raw
local_paths = [p for p in url_list if isinstance(p, str) and not _is_url(p) and os.path.isfile(p)]
url_paths = [p.strip() for p in url_list if isinstance(p, str) and _is_url(p)]
if len(url_paths) == 1 and not local_paths:
return redirect(url_paths[0], code=302)
if len(local_paths) == 1 and not url_paths:
return send_file(
local_paths[0],
as_attachment=True,
download_name=os.path.basename(local_paths[0])
)
if local_paths or url_paths:
buf = io.BytesIO()
with zipfile.ZipFile(buf, 'w', zipfile.ZIP_DEFLATED) as zf:
for p in local_paths:
zf.write(p, os.path.basename(p))
for url in url_paths:
try:
import requests as req
r = req.get(url, timeout=30, stream=True)
r.raise_for_status()
name = os.path.basename(urlparse(url).path) or ('file_%s' % (url_paths.index(url)))
if not name or name == 'file_%s' % url_paths.index(url):
name = 'download_%s' % url_paths.index(url)
zf.writestr(name, r.content)
except Exception:
pass
buf.seek(0)
return send_file(
buf,
mimetype='application/zip',
as_attachment=True,
download_name='brand_task_%s.zip' % task_id
)
return jsonify({'success': False, 'error': '结果文件不存在或链接不可用'}), 404
except Exception as e:
return jsonify({'success': False, 'error': str(e)}), 500

View File

@@ -1,82 +0,0 @@
import threading
import time
from queue import Empty, Full
from flask import Blueprint, jsonify, request
from config import JSON_TASK_QUEUE,runing_task
communication_bp = Blueprint('communication', __name__)
_del_brand_cache = {}
_del_brand_cache_lock = threading.Lock()
_latest_del_brand_item = None
_latest_del_brand_ts = 0.0
def _resolve_request_id():
rid = request.args.get('id')
if rid is not None and str(rid).strip() != '':
return str(rid)
if request.method == 'POST':
body = request.get_json(silent=True) or {}
rid = body.get('id')
if rid is not None and str(rid).strip() != '':
return str(rid)
return None
@communication_bp.route('/api/amazon/del_brand', methods=['GET', 'POST'])
def del_brand():
"""返回队列中的一个元素;同一 id 重复请求返回已缓存的同一元素2分钟内不同 id 也返回最近元素。"""
global _latest_del_brand_item, _latest_del_brand_ts
request_id = _resolve_request_id()
if not request_id:
return jsonify({'success': False, 'message': '缺少参数 id', 'data': None}), 400
with _del_brand_cache_lock:
if request_id in _del_brand_cache:
item = _del_brand_cache[request_id]
print("【删除品牌】缓存命中", request_id, item)
return jsonify({'success': True, 'data': item})
if _latest_del_brand_item is not None and (time.time() - _latest_del_brand_ts) <= 120:
_del_brand_cache[request_id] = _latest_del_brand_item
print("【删除品牌】2分钟窗口命中", request_id, _latest_del_brand_item)
return jsonify({'success': True, 'data': _latest_del_brand_item})
try:
item = JSON_TASK_QUEUE.get_nowait()
except Empty:
print("队列为空")
return jsonify({'success': False, 'message': '队列为空', 'data': None})
with _del_brand_cache_lock:
if request_id in _del_brand_cache:
try:
JSON_TASK_QUEUE.put_nowait(item)
except Full:
print("【删除品牌】队列已满,无法归还重复拉取的任务")
item = _del_brand_cache[request_id]
print("【删除品牌】并发归并", request_id, item)
return jsonify({'success': True, 'data': item})
_del_brand_cache[request_id] = item
_latest_del_brand_item = item
_latest_del_brand_ts = time.time()
print("【删除品牌】返回任务数据", item)
return jsonify({'success': True, 'data': item})
@communication_bp.route('/api/amazon/get_all_detail', methods=['GET', 'POST'])
def get_all_detail():
"""获取正在执行的删除品牌任务详情"""
request_id = _resolve_request_id()
if not request_id: #如果没有提供 id 参数,直接返回所有正在执行的任务详情
return jsonify({'success': True, 'data': runing_task})
task_info = runing_task.get(request_id)
if not task_info:
return jsonify({'success': False, 'message': '任务不存在', 'data': None})
return jsonify({'success': True, 'data': task_info})

View File

@@ -1,411 +0,0 @@
"""
图片生成蓝图:生成、历史、下载、拼接、版本
"""
import os
import sys
import json
import re
import io
import base64
import tempfile
import threading
import subprocess
import requests
from urllib.parse import urlparse, quote
from flask import Blueprint, request, jsonify, Response, session, send_file
from PIL import Image
from app_common import get_db, login_required, BASE_DIR
from config import STITCH_WORKFLOW_ID,client_name
image_bp = Blueprint('image', __name__)
def _load_image_from_url_or_data(url_or_data):
"""从 http(s) URL 或 data URL 加载为 PIL Image失败返回 None"""
if not url_or_data or not isinstance(url_or_data, str):
return None
try:
if url_or_data.startswith('data:'):
m = re.match(r'data:image/[^;]+;base64,(.+)', url_or_data, re.DOTALL)
if not m:
return None
raw = base64.b64decode(m.group(1).strip())
img = Image.open(io.BytesIO(raw))
elif url_or_data.startswith(('http://', 'https://')):
resp = requests.get(url_or_data, timeout=15)
resp.raise_for_status()
img = Image.open(io.BytesIO(resp.content))
else:
return None
if img.mode != 'RGB':
img = img.convert('RGB')
return img
except Exception:
return None
def _stitch_and_upload_long_image(urls):
"""将多张图片 URL 先上传获取 file_id再调用 workflow_run 拼接长图,返回 data.merged_image_url失败返回 None。"""
if not urls or not isinstance(urls, (list, tuple)):
return None
from coze import upload_file as coze_upload_file, workflow_run
file_ids = []
temp_paths = []
try:
for u in urls:
img = _load_image_from_url_or_data(u)
if img is None:
continue
fd, path = tempfile.mkstemp(suffix='.png')
try:
os.close(fd)
img.save(path)
temp_paths.append(path)
resp = coze_upload_file(path)
if resp.get('code') == 0 and resp.get('data', {}).get('id'):
file_ids.append(resp['data']['id'])
except Exception:
pass
if not file_ids:
return None
parameters = {"images": [{"file_id": fid} for fid in file_ids]}
resp = workflow_run(STITCH_WORKFLOW_ID, parameters, is_async=False)
if resp.get('code') != 0:
return None
data = resp.get('data') or {}
data = json.loads(data)
merged_image_url = data.get('merged_image_url')
if merged_image_url:
return merged_image_url
output_str = data.get('output') or ''
if output_str:
try:
outer = json.loads(output_str)
inner_str = outer.get('Output', '{}')
inner = json.loads(inner_str)
data_str = inner.get('data', '[]')
inner_data = json.loads(data_str)
merged_image_url = inner_data.get('merged_image_url')
return merged_image_url
except Exception:
pass
return None
except Exception:
return None
finally:
for p in temp_paths:
try:
os.unlink(p)
except Exception:
pass
def _sanitize_params_for_history(params):
"""移除 base64 大字段及敏感字段,仅保留可存储的请求参数"""
exclude = ('ref_images', 'proc_images', 'layout_image')
out = {}
for k, v in (params or {}).items():
if k == 'api_key':
continue
if k in exclude:
if isinstance(v, list):
out[f'{k}_count'] = len(v)
else:
out[f'{k}_count'] = 1 if v else 0
elif isinstance(v, (str, int, float, bool, type(None))):
out[k] = v
elif isinstance(v, list) and not v:
out[k] = []
elif isinstance(v, list) and isinstance(v[0], (str, int, float, bool)):
out[k] = v
else:
out[k] = str(v)[:200] if v else None
return out
@image_bp.route('/api/generate', methods=['POST'])
@login_required
def api_generate():
"""生成图片:调用 generate_api上传原图到 OSS保存历史记录"""
try:
params = request.get_json() or {}
from generate_api import generate
result = generate(params)
if result.get('success') and result.get('urls'):
long_image_url = result.get("long_image_url")
result["long_image_url"] = long_image_url
import json as _json
history_id = None
try:
hid = params.get('history_id')
if hid is not None:
try:
hid = int(hid)
except (TypeError, ValueError):
hid = None
conn = get_db()
with conn.cursor() as cur:
if hid is not None and hid > 0:
cur.execute(
"SELECT result_urls FROM image_history WHERE id=%s AND user_id=%s",
(hid, session['user_id']),
)
row = cur.fetchone()
existing_urls = []
if row and row.get('result_urls'):
try:
existing_urls = _json.loads(row['result_urls'])
except Exception:
existing_urls = []
new_urls = result.get('urls') or []
new_url = new_urls[0] if new_urls else None
idx = params.get('history_index', 0)
try:
idx = int(idx)
except (TypeError, ValueError):
idx = 0
if new_url:
if not isinstance(existing_urls, list):
existing_urls = []
while len(existing_urls) <= idx:
existing_urls.append(existing_urls[-1] if existing_urls else new_url)
existing_urls[idx] = new_url
merged_result_urls = existing_urls or new_urls
cur.execute(
"""UPDATE image_history SET panel_type=%s, original_urls=%s, params=%s, result_urls=%s
WHERE id=%s AND user_id=%s""",
(
params.get('panel_type', ''),
_json.dumps(result.get('original_urls') or []),
_json.dumps(_sanitize_params_for_history(params)),
_json.dumps(merged_result_urls),
hid,
session['user_id'],
),
)
if cur.rowcount > 0:
history_id = hid
else:
cur.execute(
"""INSERT INTO image_history (user_id, panel_type, original_urls, params, result_urls, long_image_url)
VALUES (%s, %s, %s, %s, %s, %s)""",
(
session['user_id'],
params.get('panel_type', ''),
_json.dumps(result.get('original_urls') or []),
_json.dumps(_sanitize_params_for_history(params)),
_json.dumps(result.get('urls') or []),
long_image_url,
),
)
history_id = cur.lastrowid
conn.commit()
conn.close()
except Exception:
pass
if history_id is not None:
result['history_id'] = history_id
return jsonify(result)
except Exception as e:
import traceback
traceback.print_exc()
return jsonify({'success': False, 'urls': [], 'error': str(e)})
@image_bp.route('/api/version')
def api_version():
"""检测更新:请求 APP_UPDATE_URL 获取最新版本信息,与当前版本比较"""
current_version = (os.environ.get('APP_VERSION', '1.0.0') or '1.0.0').strip()
update_url = (os.environ.get('APP_UPDATE_URL', '') or '').strip()
result = {
'version': current_version,
'desc': '',
'url': '',
'has_update': False,
'latest_version': current_version,
'file_url': '',
}
if not update_url:
return jsonify(result)
try:
resp = requests.get(update_url, timeout=10)
resp.raise_for_status()
data = resp.json() or {}
latest_version = (data.get('version') or '').strip()
file_url = (data.get('file_url') or '').strip()
result['latest_version'] = latest_version
result['file_url'] = file_url
result['url'] = file_url
# 版本不一致则视为有更新
if latest_version and latest_version != current_version:
result['has_update'] = True
except Exception:
pass
return jsonify(result)
def _run_update_and_exit(zip_path, target_dir):
"""在后台延迟后启动 update.exe脱离当前进程然后退出当前程序"""
def _do():
import time
time.sleep(1.5) # 确保 HTTP 响应已发送
# exe_dir = target_dir
# exe_dir = "D:\\pack\\nanri\\update_exe\\main.dist"
exe_dir = os.path.join(BASE_DIR,"update")
update_exe = os.path.join(exe_dir, 'update.exe')
if not os.path.isfile(update_exe):
return
try:
creationflags = 0
if sys.platform == 'win32':
creationflags = subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
subprocess.Popen(
[update_exe, '--zip', zip_path, '--target', target_dir, '--process',client_name],
cwd=exe_dir,
creationflags=creationflags,
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
close_fds=True,
)
except Exception:
pass
os._exit(0)
t = threading.Thread(target=_do, daemon=False)
t.start()
@image_bp.route('/api/update/do', methods=['POST'])
def api_update_do():
"""执行更新:下载 zip 到 tmp启动 update.exe 后退出程序"""
data = request.get_json() or {}
file_url = (data.get('file_url') or '').strip()
if not file_url:
return jsonify({'success': False, 'error': '缺少 file_url'}), 400
parsed = urlparse(file_url)
if parsed.scheme not in ('http', 'https'):
return jsonify({'success': False, 'error': '无效的下载地址'}), 400
tmp_dir = os.path.join(BASE_DIR, 'tmp')
try:
os.makedirs(tmp_dir, exist_ok=True)
except Exception as e:
return jsonify({'success': False, 'error': f'创建 tmp 目录失败: {e}'}), 500
# 使用 URL 中的文件名或默认版本名
filename = os.path.basename(parsed.path) or 'update.zip'
zip_path = os.path.join(tmp_dir, filename)
try:
resp = requests.get(file_url, timeout=300, stream=True)
resp.raise_for_status()
with open(zip_path, 'wb') as f:
for chunk in resp.iter_content(chunk_size=65536):
if chunk:
f.write(chunk)
except requests.RequestException as e:
return jsonify({'success': False, 'error': f'下载失败: {e}'}), 502
_run_update_and_exit(zip_path, BASE_DIR)
return jsonify({'success': True, 'message': '更新已启动,程序即将退出'})
@image_bp.route('/api/download')
@login_required
def api_download():
"""代理下载图片,解决跨域 fetch 无法下载的问题"""
url = request.args.get('url', '').strip()
filename = request.args.get('filename', 'image.png')
if not url:
return jsonify({'success': False, 'error': '缺少 url 参数'}), 400
parsed = urlparse(url)
if parsed.scheme not in ('http', 'https'):
return jsonify({'success': False, 'error': '仅支持 http/https 链接'}), 400
try:
resp = requests.get(url, timeout=30, stream=True)
resp.raise_for_status()
content_type = resp.headers.get('Content-Type', 'image/png')
encoded = quote(filename, safe='')
disposition = f"attachment; filename*=UTF-8''{encoded}"
return Response(
resp.iter_content(chunk_size=8192),
mimetype=content_type,
headers={'Content-Disposition': disposition}
)
except requests.RequestException as e:
return jsonify({'success': False, 'error': str(e)}), 502
@image_bp.route('/api/stitch/save', methods=['POST'])
@login_required
def api_stitch_save():
"""手动拼接:接收图片 URL 列表(支持 http 或 data URL拼接并上传返回长图 URL"""
try:
data = request.get_json() or {}
urls = data.get('urls')
if not urls or not isinstance(urls, list):
return jsonify({'success': False, 'error': '请提供 urls 数组'}), 400
urls = [u for u in urls if u and isinstance(u, str)]
if not urls:
return jsonify({'success': False, 'error': '没有有效的图片'}), 400
long_image_url = _stitch_and_upload_long_image(urls)
if not long_image_url:
return jsonify({'success': False, 'error': '拼接或上传失败'}), 500
return jsonify({'success': True, 'long_image_url': long_image_url})
except Exception as e:
return jsonify({'success': False, 'error': str(e)}), 500
@image_bp.route('/api/history')
@login_required
def api_history():
"""分页获取当前用户的历史图库,支持按 panel_type 栏目筛选"""
page = max(1, int(request.args.get('page', 1)))
page_size = min(50, max(10, int(request.args.get('page_size', 20))))
panel_type = (request.args.get('panel_type') or '').strip()
offset = (page - 1) * page_size
try:
conn = get_db()
with conn.cursor() as cur:
where_user = "user_id = %s"
params_where = [session['user_id']]
if panel_type:
where_user += " AND panel_type = %s"
params_where.append(panel_type)
cur.execute(
"""SELECT id, created_at, panel_type, original_urls, params, result_urls, long_image_url
FROM image_history WHERE """ + where_user + """ ORDER BY created_at DESC LIMIT %s OFFSET %s""",
params_where + [page_size, offset],
)
rows = cur.fetchall()
cur.execute("SELECT COUNT(*) as total FROM image_history WHERE " + where_user, params_where)
total = cur.fetchone()['total']
conn.close()
def _parse_json(val, default=None):
if val is None:
return default if default is not None else []
if isinstance(val, (list, dict)):
return val
try:
return json.loads(val)
except Exception:
return default if default is not None else []
items = []
for r in rows:
items.append({
'id': r['id'],
'created_at': r['created_at'].strftime('%Y-%m-%d %H:%M') if r['created_at'] else '',
'panel_type': r['panel_type'] or '',
'original_urls': _parse_json(r['original_urls'], []),
'params': _parse_json(r['params'], {}),
'result_urls': _parse_json(r['result_urls'], []),
'long_image_url': (r.get('long_image_url') or '').strip() or None,
})
return jsonify({'success': True, 'items': items, 'total': total, 'page': page, 'page_size': page_size})
except Exception as e:
return jsonify({'success': False, 'error': str(e)})

View File

@@ -1,270 +0,0 @@
"""
主页面蓝图首页、home、图片工作台、品牌页、静态文件、Logo
"""
import os
from flask import Blueprint, send_file, render_template_string
from app_common import (
get_db,
_render_html,
_is_session_user_valid,
login_required,
admin_required,
STATIC_DIR,
BASE_DIR,
ASSETS_DIR
)
from flask import redirect, url_for, session
from flask import Flask, request, Response, stream_with_context
import requests
from config import base_url,version,JAVA_API_BASE
main_bp = Blueprint('main', __name__)
def _resolve_asset_filename(filename):
"""Resolve hashed Vite assets without maintaining hardcoded alias tables."""
safe_name = os.path.basename(filename)
asset_dirs = [
ASSETS_DIR,
os.path.abspath(os.path.join(BASE_DIR, 'new_web_source', 'assets')),
os.path.abspath(os.path.join(BASE_DIR, '..', 'new_web_source', 'assets')),
]
for asset_dir in asset_dirs:
exact_path = os.path.join(asset_dir, safe_name)
if os.path.isfile(exact_path):
return safe_name
base_name, ext = os.path.splitext(safe_name)
if not base_name or not ext:
return safe_name
parts = base_name.split('-')
if len(parts) < 2:
return safe_name
asset_names = []
for asset_dir in asset_dirs:
try:
asset_names.extend(os.listdir(asset_dir))
except OSError:
continue
if not asset_names:
return safe_name
for prefix_length in range(len(parts) - 1, 0, -1):
prefix = '-'.join(parts[:prefix_length])
matches = [
asset_name for asset_name in asset_names
if asset_name.endswith(ext) and asset_name.startswith(prefix)
]
if len(matches) == 1:
return matches[0]
return safe_name
@main_bp.route('/')
def index():
if session.get('user_id') and _is_session_user_valid():
return redirect(url_for('main.home'))
return redirect(url_for('auth.login'))
@main_bp.route('/home')
@login_required
def home():
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute("SELECT username, is_admin FROM users WHERE id = %s", (session['user_id'],))
row = cur.fetchone()
conn.close()
return _render_html('home.html', username=row.get('username', ''), is_admin=bool(row.get('is_admin')), user_id=session.get('user_id'),baseUrl=base_url,version=version)
except Exception:
return _render_html('home.html', username=session.get('username', ''), is_admin=False, user_id=session.get('user_id'),baseUrl=base_url,version=version)
@main_bp.route('/image')
@login_required
def wb():
return _render_html('index.html')
@main_bp.route('/brand')
@login_required
def brand_page():
candidate_paths = [
os.path.join(BASE_DIR, 'web_source', 'brand.html'),
os.path.abspath(os.path.join(BASE_DIR, '..', 'web_source', 'brand.html')),
]
for candidate_path in candidate_paths:
if not os.path.isfile(candidate_path):
continue
try:
with open(candidate_path, 'rb') as f:
raw = f.read()
try:
from html_crypto import decrypt
content = decrypt(raw).decode('utf-8')
except Exception:
if raw[:7] == b'gAAAAAB':
raise
content = raw.decode('utf-8', errors='replace')
return render_template_string(content, user_id=session.get('user_id'))
except Exception:
continue
return _render_html('brand.html', user_id=session.get('user_id'))
@main_bp.route('/brand/legacy')
@login_required
def brand_page_legacy():
legacy_path = os.path.join(BASE_DIR, 'web_source', 'templates_backup', 'brand.html')
if not os.path.isfile(legacy_path):
return '', 404
with open(legacy_path, 'rb') as f:
content = f.read().decode('utf-8', errors='replace')
content = content.replace(
'<header class="top-bar">',
'<header class="top-bar" style="display:none !important;">',
1,
)
content = content.replace('height: calc(100vh - 56px);', 'height: 100vh;', 1)
return render_template_string(content, user_id=session.get('user_id'))
@main_bp.route('/static/<path:filename>')
def serve_static(filename):
"""提供 static 目录及子目录下的静态文件访问。"""
filepath = os.path.normpath(os.path.join(STATIC_DIR, filename))
static_abs = os.path.abspath(STATIC_DIR)
file_abs = os.path.abspath(filepath)
if not file_abs.startswith(static_abs) or not os.path.isfile(file_abs):
return '', 404
return send_file(file_abs, as_attachment=False)
@main_bp.route('/assets/<path:filename>')
def serve_assets(filename):
"""提供 static 目录及子目录下的静态文件访问。"""
filename = _resolve_asset_filename(filename)
candidate_dirs = [
os.path.abspath(ASSETS_DIR),
os.path.abspath(os.path.join(BASE_DIR, 'new_web_source', 'assets')),
os.path.abspath(os.path.join(BASE_DIR, '..', 'new_web_source', 'assets')),
]
for static_abs in candidate_dirs:
filepath = os.path.normpath(os.path.join(static_abs, filename))
file_abs = os.path.abspath(filepath)
if not file_abs.startswith(static_abs):
continue
if os.path.isfile(file_abs):
return send_file(file_abs, as_attachment=False)
return '', 404
@main_bp.route('/new_web_source/<path:filename>')
@login_required
def serve_new_web_source(filename):
"""提供 static 目录及子目录下的静态文件访问。"""
filepath = os.path.normpath(os.path.join("new_web_source", filename))
static_abs = os.path.abspath("new_web_source")
file_abs = os.path.abspath(filepath)
if not file_abs.startswith(static_abs) or not os.path.isfile(file_abs):
return '', 404
if filename.lower().endswith('.html'):
with open(file_abs, 'rb') as f:
content = f.read().decode('utf-8', errors='replace')
raw_uid = session.get('user_id')
uid_value = str(int(raw_uid)) if str(raw_uid or '').isdigit() else '""'
uid_script = (
"\n<script>"
f"window.localStorage.setItem('uid', {uid_value});"
"</script>\n"
)
if '</head>' in content:
content = content.replace('</head>', uid_script + '</head>', 1)
else:
content = uid_script + content
return render_template_string(content)
return send_file(file_abs, as_attachment=False)
@main_bp.route('/logo.jpg', methods=['GET'])
def get_logo_image():
return send_file(os.path.join(BASE_DIR, "logo.jpg"), mimetype='image/jpeg')
@main_bp.route('/newApi/<path:path>', methods=['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'])
def proxy(path):
target_url = f"{JAVA_API_BASE}/{path}"
# 复制请求参数
params = request.args.to_dict()
# 处理请求头
headers = {}
for key, value in request.headers:
if key.lower() in ['host', 'content-length', 'connection']:
continue
headers[key] = value
ignore_url = [f"{JAVA_API_BASE}/api/delete-brand/tasks/batch",
f"{JAVA_API_BASE}/api/delete-brand/tasks/progress/batch",
f"{JAVA_API_BASE}/api/delete-brand/history",
f"{JAVA_API_BASE}/api/product-risk-resolve/tasks/batch",
f"{JAVA_API_BASE}/api/product-risk-resolve/tasks/progress/batch",
]
proxy_timeout = 180 if target_url.endswith("/api/delete-brand/run") else 30
if target_url not in ignore_url:
try:
print("=============================")
print("target_url", target_url)
print("params", params)
print("data", request.get_data())
print("method", request.method)
print("timeout", proxy_timeout)
print("=============================")
except Exception as e:
print("打印失败", e)
try:
# 使用流式请求
req = requests.request(
method=request.method,
url=target_url,
params=params,
headers=headers,
data=request.get_data() if request.get_data() else None,
cookies=request.cookies,
stream=True, # 启用流式传输
timeout=proxy_timeout
)
# 流式响应
def generate():
for chunk in req.iter_content(chunk_size=8192):
if chunk:
yield chunk
# 构建响应
response = Response(stream_with_context(generate()), status=req.status_code)
# 复制响应头
for key, value in req.headers.items():
if key.lower() not in ['content-encoding', 'content-length', 'transfer-encoding', 'connection']:
response.headers[key] = value
return response
except requests.exceptions.Timeout:
print(f"后端服务超时target_url={target_url}, timeout={proxy_timeout}")
return Response("后端服务超时", status=504)
except requests.exceptions.ConnectionError as e:
print(f"无法连接到后端服务target_url={target_url}, error={e}")
return Response("无法连接到后端服务", status=502)
except Exception as e:
print(f"代理请求失败target_url={target_url}, error={e}")
return Response(f"代理错误: {str(e)}", status=500)

View File

@@ -1,604 +0,0 @@
import json
import re
import random
import requests
import time
import unicodedata
from brand_spider.web_dec import decrypt_via_service, get_guid
try:
from config import proxy_url as CONFIG_PROXY_URL, proxy_mode as CONFIG_PROXY_MODE
except ImportError:
CONFIG_PROXY_URL = None
CONFIG_PROXY_MODE = 1
# Forbidden 后 3 分钟内统一使用代理:记录代理生效截止时间与当前代理
_FORBIDDEN_PROXY_UNTIL = 0.0
_FORBIDDEN_PROXY_DICT = None
_FORBIDDEN_PROXY_MINUTES = 0.5
special_char_pattern = re.compile(r'[^\w\s]')
def remove_accents(input_str):
# 将字符分解为基础字符和重音符号
nksel = unicodedata.normalize('NFKD', input_str)
# 过滤掉非间距重音符号,并重新编码
return "".join([c for c in nksel if not unicodedata.combining(c)])
def clean_text(text):
text = remove_accents(text)
return special_char_pattern.sub(' ', text).replace(" ","")
def create_code_generator():
counter = random.randint(0, 0xFFFFFF)
def get_code():
nonlocal counter
counter = (counter + 1) % 0xFFFFFF
low_16_bits = counter & 0xFFFF
return f"{low_16_bits:04x}"
return get_code()
def search(hashsearch, data, proxies=None):
global _FORBIDDEN_PROXY_UNTIL, _FORBIDDEN_PROXY_DICT
url = "https://api.branddb.wipo.int/search"
headers = {
"accept": "application/json, text/plain, */*",
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
"cache-control": "no-cache",
"content-type": "application/json",
"hashsearch": hashsearch,
"origin": "https://branddb.wipo.int",
"pragma": "no-cache",
"priority": "u=1, i",
"referer": "https://branddb.wipo.int/",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"
}
# 若 3 分钟内曾出现 Forbidden则直接使用当时保存的代理
now = time.time()
if now < _FORBIDDEN_PROXY_UNTIL and _FORBIDDEN_PROXY_DICT:
proxies = _FORBIDDEN_PROXY_DICT
print("处于 Forbidden 代理窗口内,直接使用代理",proxies)
# 发送 POST 请求
response = requests.post(url, headers=headers, json=data, proxies=proxies)
enc_text = response.text
# print("原始结果", enc_text)
# 若返回 Forbidden则从 config 的 proxy_url 获取代理 IP 后重试,并开启 3 分钟代理窗口
if (enc_text.strip() == '{"message":"Forbidden"}' or "Too Many Requests" in enc_text) and CONFIG_PROXY_URL:
try:
proxy_resp = requests.get(CONFIG_PROXY_URL, timeout=10)
print("代理请求结果->", proxy_resp.text)
# 模式 2账号密码代理接口返回 JSON
if CONFIG_PROXY_MODE == 2:
resp_json = proxy_resp.json()
proxy_list = resp_json.get("data", {}).get("list") or []
first_item = proxy_list[0] if proxy_list else None
if first_item:
ip = first_item.get("ip")
port = first_item.get("port")
account = first_item.get("account")
password = first_item.get("password")
if ip and port and account and password:
auth_proxy = f"{account}:{password}@{ip}:{port}"
print("获取到账号密码代理:", auth_proxy)
proxies = {
"http": f"http://{auth_proxy}",
"https": f"http://{auth_proxy}",
}
# 默认模式 1普通 IP:port 文本
if CONFIG_PROXY_MODE != 2:
proxy_ip = (proxy_resp.text or "").strip()
if proxy_ip:
proxies = {
"http": f"http://{proxy_ip}",
"https": f"http://{proxy_ip}",
}
if proxies:
response = requests.post(url, headers=headers, json=data, proxies=proxies)
enc_text = response.text
print("代理重试结果", enc_text)
# 记录 3 分钟内都使用该代理
_FORBIDDEN_PROXY_UNTIL = now + _FORBIDDEN_PROXY_MINUTES * 60
_FORBIDDEN_PROXY_DICT = proxies
except Exception as e:
print("获取代理或重试失败:", e)
return enc_text
class TaskCancelledError(Exception):
"""用户取消任务时抛出,供上层捕获并中止"""
pass
def single_file_handle(brand, strategy="Terms"):
status_info = {
"Ended": "已结束",
"Expired": "已过期的",
"Pending": "待决",
"Registered": "已注册",
"RegisteredMadrid": "国际注册有效",
"Unknown": "未知"
}
country_info = {
"AB": "ARABPAT",
"AD": "安道尔",
"AE": "UAE",
"AF": "阿富汗",
"AFR": "非洲",
"AG": "安提瓜和巴布达",
"AI": "安圭拉",
"AL": "阿尔巴尼亚",
"AM": "亚美尼亚",
"AN": "荷属安的列斯",
"ANT": "南极洲",
"AO": "安哥拉",
"AP": "非洲地区知识产权组织 ",
"AQ": "南极洲",
"AR": "阿根廷",
"AS": "美属萨摩亚",
"ASI": "亚洲",
"AT": "奥地利",
"AU": "澳大利亚",
"AW": "阿鲁巴",
"AX": "奥兰群岛",
"AZ": "阿塞拜疆",
"BA": "波斯尼亚和黑塞哥维纳",
"BB": "巴巴多斯",
"BD": "孟加拉国",
"BE": "比利时",
"BF": "布基纳法索",
"BG": "保加利亚",
"BH": "巴林",
"BI": "布隆迪",
"BJ": "贝宁",
"BL": "圣巴泰勒米",
"BM": "百慕大",
"BN": "文莱达鲁萨兰国",
"BO": "多民族玻利维亚国",
"BQ": "博纳尔、圣俄斯塔休斯和萨巴",
"BR": "巴西",
"BS": "巴哈马",
"BT": "不丹",
"BV": "布韦岛",
"BW": "博茨瓦纳",
"BX": "比荷卢知识产权局",
"BY": "白俄罗斯",
"BZ": "伯利兹",
"CA": "加拿大",
"CC": "科科斯群岛(基灵群岛)",
"CD": "刚果民主共和国",
"CF": "中非共和国",
"CG": "刚果",
"CH": "瑞士",
"CI": "科特迪瓦",
"CK": "库克群岛",
"CL": "智利",
"CM": "喀麦隆",
"CN": "中国",
"CO": "哥伦比亚",
"CR": "哥斯达黎加",
"CS": "捷克斯洛伐克",
"CU": "古巴",
"CV": "佛得角",
"CW": "库拉索",
"CX": "圣诞岛",
"CY": "塞浦路斯",
"CZ": "捷克共和国",
"DD": "德意志民主共和国",
"DE": "德国",
"DJ": "吉布提",
"DK": "丹麦",
"DM": "多米尼克",
"DO": "多米尼加",
"DT": "西德",
"DZ": "阿尔及利亚",
"EA": "欧亚专利组织",
"EC": "厄瓜多尔",
"EE": "爱沙尼亚",
"EG": "埃及",
"EH": "西撒哈拉",
"EM": "欧洲联盟",
"EP": "欧洲专利局",
"ER": "厄立特里亚",
"ES": "西班牙",
"ET": "埃塞俄比亚",
"EUR": "欧洲",
"FI": "芬兰",
"FJ": "斐济",
"FK": "福克兰群岛(马尔维纳斯群岛)",
"FM": "密克罗尼西亚联邦",
"FO": "法罗群岛",
"FR": "法国",
"GA": "加蓬",
"GB": "英国",
"GC": "海湾阿拉伯国家合作委员会专利局",
"GD": "格林纳达",
"GE": "格鲁吉亚",
"GF": "法属圭亚那",
"GG": "格恩西岛",
"GH": "加纳",
"GI": "直布罗陀",
"GL": "格陵兰",
"GM": "冈比亚",
"GN": "几内亚",
"GP": "瓜德罗普",
"GQ": "赤道几内亚",
"GR": "希腊",
"GS": "南乔治亚和南桑威奇群岛",
"GT": "危地马拉",
"GU": "关岛",
"GW": "几内亚比绍",
"GY": "圭亚那",
"HK": "香港",
"HM": "赫德岛和麦克唐纳群岛",
"HN": "洪都拉斯",
"HR": "克罗地亚",
"HT": "海地",
"HU": "匈牙利",
"IB": "世界知识产权组织国际局",
"ID": "印度尼西亚",
"IE": "爱尔兰",
"IL": "以色列",
"IM": "马恩岛",
"IN": "印度",
"INN": "世界卫生组织",
"IO": "英属印度洋领地",
"IQ": "伊拉克",
"IR": "伊朗伊斯兰共和国",
"IS": "冰岛",
"IT": "意大利",
"JE": "泽西岛",
"JM": "牙买加",
"JO": "约旦",
"JP": "日本",
"KE": "肯尼亚",
"KG": "吉尔吉斯斯坦",
"KH": "柬埔寨",
"KI": "基里巴斯",
"KM": "科摩罗",
"KN": "圣基茨和尼维斯",
"KP": "朝鲜民主主义人民共和国",
"KR": "大韩民国",
"KW": "科威特",
"KY": "开曼群岛",
"KZ": "哈萨克斯坦",
"LA": "老挝人民民主共和国",
"LB": "黎巴嫩",
"LC": "圣卢西亚",
"LI": "列支敦士登",
"LISBON": "WIPO",
"LK": "斯里兰卡",
"LP": "LATIPAT",
"LR": "利比里亚",
"LS": "莱索托",
"LT": "立陶宛",
"LU": "卢森堡",
"LV": "拉脱维亚",
"LY": "利比亚",
"MA": "摩洛哥",
"MC": "摩纳哥",
"MD": "摩尔多瓦共和国",
"ME": "黑山",
"MF": "圣马丁(法国部分)",
"MG": "马达加斯加",
"MH": "马绍尔群岛",
"MK": "北马其顿共和国",
"ML": "马里",
"MM": "缅甸",
"MN": "蒙古",
"MO": "澳门",
"MP": "北马里亚纳群岛",
"MQ": "马提尼克",
"MR": "毛里塔尼亚",
"MS": "蒙特塞拉特",
"MT": "马耳他",
"MU": "毛里求斯",
"MV": "马尔代夫",
"MW": "马拉维",
"MX": "墨西哥",
"MY": "马来西亚",
"MZ": "莫桑比克",
"NA": "纳米比亚",
"NAM": "北美洲",
"NC": "新喀里多尼亚",
"NE": "尼日尔",
"NF": "诺福克岛",
"NG": "尼日利亚",
"NI": "尼加拉瓜",
"NL": "荷兰",
"NO": "挪威",
"NP": "尼泊尔",
"NR": "瑙鲁",
"NU": "纽埃",
"NZ": "新西兰",
"OA": "非洲知识产权组织",
"OCE": "大洋洲",
"OM": "阿曼",
"PA": "巴拿马",
"PE": "秘鲁",
"PF": "法属波利尼西亚",
"PG": "巴布亚新几内亚",
"PH": "菲律宾",
"PK": "巴基斯坦",
"PL": "波兰",
"PM": "圣皮埃尔和密克隆",
"PN": "皮特凯恩",
"PR": "波多黎各",
"PS": "巴勒斯坦",
"PT": "葡萄牙",
"PW": "帕劳",
"PY": "巴拉圭",
"QA": "卡塔尔",
"QO": "没有ST.3代码的组织",
"QZ": "欧洲联盟",
"RE": "留尼汪",
"RO": "罗马尼亚",
"RS": "塞尔维亚",
"RU": "俄罗斯联邦",
"RW": "卢旺达",
"SA": "沙特阿拉伯",
"SAM": "南美洲",
"SB": "所罗门群岛",
"SC": "塞舌尔",
"SD": "苏丹",
"SE": "瑞典",
"SG": "新加坡",
"SH": "圣赫勒拿、阿森松和特里斯坦-达库尼亚",
"SI": "斯洛文尼亚",
"SIXTER": "WIPO",
"SJ": "斯瓦尔巴和扬马延",
"SK": "斯洛伐克",
"SL": "塞拉里昂",
"SM": "圣马力诺",
"SN": "塞内加尔",
"SO": "索马里",
"SR": "苏里南",
"SS": "南苏丹",
"ST": "圣多美和普林西比",
"SU": "苏联",
"SV": "萨尔瓦多",
"SX": "圣马丁(荷兰部分)",
"SY": "阿拉伯叙利亚共和国",
"SZ": "斯威士兰",
"TC": "特克斯和凯科斯群岛",
"TD": "乍得",
"TF": "法属南部领地",
"TG": "多哥",
"TH": "泰国",
"TJ": "塔吉克斯坦",
"TK": "托克劳",
"TL": "东帝汶",
"TM": "土库曼斯坦",
"TN": "突尼斯",
"TO": "汤加",
"TR": "土耳其",
"TT": "特立尼达和多巴哥",
"TV": "图瓦卢",
"TW": "台湾(中国的省)",
"TZ": "坦桑尼亚联合共和国",
"UA": "乌克兰",
"UG": "乌干达",
"UK": "UK",
"UM": "美国本土外小岛屿",
"US": "USA",
"UY": "乌拉圭",
"UZ": "乌兹别克斯坦",
"VA": "罗马教廷",
"VC": "圣文森特和格林纳丁斯",
"VE": "委内瑞拉玻利瓦尔共和国",
"VG": "英属维尔京群岛",
"VI": "美属维尔京群岛",
"VN": "越南",
"VU": "瓦努阿图",
"WF": "瓦利斯和富图纳",
"WHO": "世界卫生组织",
"WO": "WIPO",
"WS": "萨摩亚",
"XK": "科索沃共和国",
"XN": "北欧专利局",
"XX": "国际",
"XXX": "跨国和国际局",
"YD": "民主也门",
"YE": "也门",
"YT": "马约特岛",
"YU": "塞尔维亚和黑山",
"ZA": "南非",
"ZM": "赞比亚",
"ZW": "津巴布韦"
}
check_value = [
"瑞典",
"芬兰",
"丹麦",
"挪威",
"冰岛",
"法国",
"爱尔兰",
"荷兰",
"比利时",
"卢森堡",
"英国",
"摩纳哥",
"德国",
"波兰",
"捷克",
"斯洛伐克",
"匈牙利",
"奥地利",
"瑞士",
"列支敦士登",
"爱沙尼亚",
"拉脱维亚",
"立陶宛",
"俄罗斯",
"白俄罗斯",
"乌克兰",
"摩尔多瓦",
"西班牙",
"葡萄牙",
"意大利",
"希腊",
"斯洛文尼亚",
"克罗地亚",
"罗马尼亚",
"保加利亚",
"塞尔维亚",
"阿尔巴尼亚",
"黑山",
"马耳他",
"塞浦路斯",
"北马其顿",
"梵蒂冈",
"圣马力诺",
"安道尔",
"波黑",
"欧洲联盟"
]
check_staus = ["已过期的","已结束"]
# 5. 初始化不符合品牌的数据列表
faild_data = []
# 初始化查询失败的品牌的数据列表
query_faild_data = []
# Simple 策略下最多翻 3 页0,30,60直到命中 554 行判断
max_pages = 3 if strategy == "Simple" else 1
page = 0
matched = False
while page < max_pages and not matched:
try:
as_structure_dict = {
"_id": create_code_generator(),
"boolean": "AND",
"bricks": [
{
"_id": create_code_generator(),
"key": "brandName",
"value": brand,
"strategy": strategy
}
]
}
# 构造 asStructure 内部 JSON 对象
as_structure_dict = as_structure_dict
# print(as_structure_dict)
# 将内部对象转为 JSON 字符串(作为 asStructure 字段的值)
as_structure_str = json.dumps(as_structure_dict, ensure_ascii=False)
data = {
"sort": "score desc",
"rows": "30",
"asStructure": as_structure_str,
"fg": "_void_"
}
# 翻页:第二页 start=30第三页 start=60仅 Simple 策略)
if strategy == "Simple" and page > 0:
data["start"] = str(30 * page)
# print("请求参数", as_structure_dict, "页码:", page)
hashsearch = get_guid()
enc_text = search(hashsearch, data)
print(f"{hashsearch}】待解密-->", enc_text)
dec_text = decrypt_via_service(hashsearch, enc_text)
# print(type(dec_text))
dec_text = str(dec_text)
# print(f"解密之后的结果-->", dec_text)
data = json.loads(dec_text)
except Exception as e:
# 仅第一页请求失败时记录为查询失败品牌
if page == 0:
import traceback
traceback.print_exc()
safe_brand = brand.encode('gbk', errors='replace').decode('gbk')
print(f"品牌:{safe_brand},处理失败:{e}")
# query_faild_data.append({"brand": brand, "time": datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S")})
query_faild_data.append(brand)
break
# print(data)
if data.get("response", {}).get("numFound", 0) > 0:
if strategy == "Simple":
new_brand_name = clean_text(brand)
_brand = new_brand_name.lower()
else:
_brand = brand
docs = data["response"]["docs"]
for d in docs:
office = d.get("office")
status = d.get("status")
registrationDate = d.get("registrationDate")
status_name = status_info.get(status, status)
brand_name = d.get("brandName")
if isinstance(brand_name, list):
brand_name = "|".join(brand_name)
if strategy == "Simple":
new_brand_name = clean_text(brand_name)
new_brand_name = new_brand_name.lower()
if _brand != new_brand_name:
continue
check_office = d.get("designation")
check_office.append(office)
for i in set(check_office):
office_name = country_info.get(i, i)
# print(office_name,office_name in check_value and status_name not in check_staus)
if office_name in check_value and status_name not in check_staus:
# 命中 554 行判断,记录并结束当前品牌后续翻页
# data_rows = [row for row in data_rows if row.get("品牌") != brand]
# print(office_name, office in ["EM","QZ"] and i not in ["EM","QZ"])
if office in ["EM","QZ"] and i not in ["EM","QZ"]:
continue
if strategy == "Simple":
faild_data.append({"brand": brand, "country": office_name, "status": status_name}) # "reason":"|".join(d.get("brandName","")
matched = True
else:
# print("添加",brand,office_name)
faild_data.append({"brand": brand, "country": office_name, "status": status_name})
# break
if matched:
break
# 如果 Simple 策略下本页未命中,则翻下一页;其他策略不翻页
if not matched:
page += 1
return faild_data,query_faild_data
if __name__ == '__main__':
file_path = r"D:\\私单交付\\maixiang_AI\\测试图片数据\\品牌采集测试\\brand_task_73\\源文件\\brand_task_75\\源文件\\3.10.xlsx"
res = single_file_handle(file_path,"结果.xlsx")
print(res)

View File

@@ -1,126 +0,0 @@
"""
通过 pywebview 加载本地 HTML执行 JS 实现解密与 GUID 生成。
"""
import json
import threading
import traceback
import webview
# 内嵌 HTML加载 CryptoJS 并定义 decryptWithHashSearches、guid
_DECRYPT_HTML = """
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<script src="https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.1.1/crypto-js.min.js"></script>
</head>
<body>
<script>
function decryptWithHashSearches(ciphertext, hashSearches) {
try {
var baseKey = "8?)i_~Nk6qv0IX;2";
var keyStr = baseKey + (hashSearches || "");
var key = CryptoJS.enc.Utf8.parse(keyStr);
var decrypted = CryptoJS.AES.decrypt(ciphertext, key, {
mode: CryptoJS.mode.ECB
});
return decrypted.toString(CryptoJS.enc.Utf8);
} catch (error) {
console.error('解密失败:', error.message);
return '';
}
}
function guid() {
function _p8(s) {
var p = (Math.random().toString(16) + "000000000").substr(2, 8);
return s ? "-" + p.substr(0, 4) + "-" + p.substr(4, 4) : p;
}
return _p8() + _p8(true) + _p8(true) + _p8();
}
</script>
</body>
</html>
"""
_window = None
_window_ready = threading.Event()
_init_lock = threading.Lock()
def _ensure_window():
"""首次调用时在后台线程启动 pywebview 并等待页面加载完成。"""
global _window
with _init_lock:
if _window is not None:
return
_window = webview.create_window(
"",
html=_DECRYPT_HTML,
width=1,
height=1,
hidden=True,
)
def run():
webview.start(debug=False)
t = threading.Thread(target=run, daemon=True)
t.start()
_window.events.loaded.wait()
_window_ready.set()
_window_ready.wait()
def decrypt_via_service(hash_searches, ciphertext):
"""
通过 pywebview 执行 JS decryptWithHashSearches 进行 AES-ECB 解密。
:param hash_searches: 对应 JS 的 hashSearches密钥后缀可为空串
:param ciphertext: Base64 密文
:return: 解密后的 UTF-8 字符串,失败返回空串
"""
_ensure_window()
# 将参数安全注入 JS避免注入与引号问题
ciphertext_js = json.dumps(ciphertext)
hash_searches_js = json.dumps(hash_searches or "")
js = f"(function(){{ return decryptWithHashSearches({ciphertext_js}, {hash_searches_js}); }})();"
try:
result = _window.evaluate_js(js)
return result if result is not None else ""
except Exception as e:
raise RuntimeError(f"解密失败: {e}") from e
def get_guid():
"""通过 pywebview 执行 JS guid() 生成 UUID 格式字符串。"""
_ensure_window()
try:
result = _window.evaluate_js("(function(){ return guid(); })();")
if result is None:
raise RuntimeError("guid() 返回为空")
return result
except Exception as e:
raise RuntimeError(f"获取 GUID 失败: {e}") from e
# 使用示例
if __name__ == "__main__":
# 参数含义hash_searches 对应原 keyciphertext 对应原 data
hash_searches = "a6efb809-b714-7efd-4b64-8c650b9030f0"
ciphertext = "SEsfpOGa8B+sWk0ncknTNh/HNHbGiEVi/RNKxBvyHmAE5VjHonPi202c6VicC/GKfA8mLsIC5mGEpSaH2DdCaEJKeOTWD9SBHHbgtyS1O60VqjgAaptYe9LivvWKc/BU8sZOqhxPMzGUHDcKUts7d0p+hCc80XCXyM2ZT80smM1twndcDfpGkLDk2kJbzl2bGzIc60sl9MzKWfZA2sE0ztFjQ9wD2uhn5LrwoN8NnpiPLNbviMMGtHh4N6Dc0xtPzkzgfkiuxBfWnN1SeM9XVgujHvAGea/dqUWIJqLo26fZIOEFJ0MYL4c8CGLYIeP/70cT2IqJdf+IPBWsCiP29zSyAiQd3yLNvd8+xBLky7lR4ng3MZizn/vhW/5BSg1FtVglbAmNbKgHOIbtpP197Lv+uahx2IpsSPqpy4j2O2VV25YzBk9JpJ4WGG/SvF3f2ZYKKepEQ+kGmBxAfG/5Z8DTNIwnOpXhyjFpJb+fUdPV6LTCK/yFc8g31bNFAJkLW7y/VjewjFravZ3VfQjNAHCvifnrIxGG22MZ3TLVnlbh6ye6vTnd+v9GzqXu+ISR7vQGL/ZSM/bJIjuVkP2XnNUPf+NFdt75gyDmTylFmmbpg7WHaBjinPcyVjeZ38+quJhT8yEk66BOjBM47mVdfU8JLK3ToghxQ54dKWGUbc9HRzYIAQ0rIBBExcOcPM4J9DpufvajmEygoDaws4CxOQDlFoFLwNBYorJsKzAoDe1Cu8oFtk4x190Vd+leRKq6DQSNJwEmyrVkWyFpiuywSMaixFlSqve0lRs50FclXfV7gkBAAvz/DJp90i9yCJdMaihP5ZCvbhKxFGMowkU+tx5Ptnxd9hq6tUxHmuiwDI1eyY8EyjB+3MbrC3H/GY39Z5Qhkj5tkSBm4oGA/h6YjeunBlfMU/QGP3hyZIYALh/YmBlZxAcglWwcqlISUhcx1L3Dbw6ZQbpAmYHmA421xIlRJkdJuPmGoopcoFEVdIO0Ov3uP2JL5ybvUgCBMnbwcYRK9qRmvn2b8jHKch55YXerHBDCEMHz88rfgNIEL8DIAp+wMuIyJ0VrV0BwAXioLYNvMcggBp06gghq2NykGiXRZ/dz8nffFqPVkDNjzcPSGrml9fpKdO8x/GannMNBsA+oHlO2/d4dyo+Q2dZCarQ5UB/N1p+UsTW90kvITKwCbZfV/YY2NRm7HrXq+wLS1vY4mChY82Ybc3cYQT653Q=="
try:
decrypted_text = decrypt_via_service(hash_searches, ciphertext)
print("解密结果:", decrypted_text)
except Exception as e:
print("错误:", e)
try:
g = get_guid()
print("GUID:", g)
except Exception as e:
print("GUID 错误:", e)

View File

@@ -1,104 +0,0 @@
base_url = "https://api.coze.cn/v1"
coze_token = "sat_12nW40INoJxArrDXbY4lSCoudbqkOYTcphC99BP2efWyzxmsk4q81WDX3ezWgqZ5"
workflow_id = "7608812635877900322"
STITCH_WORKFLOW_ID = "7608813873483300907"
import os
from dotenv import load_dotenv
from queue import Queue
base_dir = os.path.dirname(os.path.abspath(__file__))
dotenv_path = os.path.join(base_dir, ".env")
load_dotenv(dotenv_path=dotenv_path, override=True)
_base_url = os.getenv("base_url","http://159.75.121.33:15124")
# MySQL 配置
mysql_host = os.getenv("mysql_host")
mysql_user = os.getenv("mysql_user")
mysql_password = os.getenv("mysql_password","WTFrb5y6hNLz6hNy")
mysql_database = os.getenv("mysql_database","aiimage")
proxy_url = os.getenv("proxy_url")
proxy_mode = int(os.getenv("proxy_mode",1))
client_name=os.getenv("client_name") + ".exe"
JAVA_API_BASE = os.getenv("java_api_base", "http://127.0.0.1:18080")
# 紫鸟浏览器配置
from urllib.parse import unquote
ZN_COMPANY = os.getenv("zn_company", "")
ZN_USERNAME = os.getenv("zn_username", "")
ZN_USERNAME = unquote(ZN_USERNAME)
ZN_PASSWORD = os.getenv("zn_password", "#20zsg25")
# 删除品牌API配置
DELETE_BRAND_API_BASE = os.getenv("DELETE_BRAND_API_BASE", JAVA_API_BASE)
cache_path = "./user_data"
region = "cn-hangzhou"
endpoint = "oss-cn-hangzhou.aliyuncs.com"
bucket = "nanri-ai-images"
accessKeyId = "LTAI5tNpyvzMNz9f2dHarsm8"
accessKeySecret = "bQSZnFH455i8tzyOgeahJmUzwmhynz"
bucket_path = "nanri-image/"
file_url_pre = f"https://{bucket}.oss-cn-hangzhou.aliyuncs.com/"
os.environ['OSS_ACCESS_KEY_ID'] = accessKeyId
os.environ['OSS_ACCESS_KEY_SECRET'] = accessKeySecret
os.environ['SECRET_KEY'] = "ddffc7c1d02121d9554d7b080b2511b6"
debug = True
version = "1.0.13"
APP_UPDATE_URL = f"{_base_url}/api/version/latest" # 检测更新接口地址,返回 { "file_url": "...", "version": "x.x.x" }
os.environ['APP_VERSION'] = version
os.environ['APP_UPDATE_URL'] = APP_UPDATE_URL
# 队列
JSON_TASK_QUEUE = Queue()
base_dir = os.getcwd()
#删除品牌,正在执行中的任务
runing_task = {
}
# 正在执行中的店铺
runing_shop = {}
#
# runing_task = {
# "497": {
# "status": "running", # running/completed/failed
# "start_time": "2026-04-01 10:00:00",
# "stop_requested": False,
# "current_shop": "魏振峰",
# "current_country": "德国",
# "current_asin": "B0D451RQRG",
# "total_shops": 1,
# "processed_shops": 0,
# "total_asins": 14,
# "processed_asins": 3,
# "success_count": 2,
# "failed_count": 1
# }
# }

View File

@@ -1,92 +0,0 @@
import requests
import time
from config import base_url, coze_token
def upload_file(file_path,_coze_token =coze_token):
url = f"{base_url}/files/upload"
headers = {
"Authorization": f"Bearer {_coze_token}"
}
print(file_path)
with open(file_path, 'rb') as f:
files = {
'file': f
}
# with open(f"{time.time()".replace(".","_")+".png","wb") as file:
# file.write(f.read())
# allow_redirects=True 模拟 curl 的 --location默认即为 True
response = requests.post(url, headers=headers, files=files, allow_redirects=True)
data = response.json()
print("上传图片->>",data)
return data
def workflow_run(workflow_id, parameters,_coze_token = coze_token,is_async=True):
url = f"{base_url}/workflow/run"
headers = {
"Authorization": f"Bearer {_coze_token}",
"Content-Type": "application/json"
}
# 请求体JSON 格式)
payload = {
"parameters": parameters,
# "parameters": {
# "name": "包包",
# "ratio": "3:4",
# "menu": 3,
# "resolution": "2K",
# "count": 1,
# "desc": "1、一个中国美女手上挎着这个包2、站在商场内",
# "language": "中文",
# "style": "极简高级"
# },
"workflow_id": workflow_id,
"is_async" : is_async,
}
# 发送 POST 请求(使用 json 参数自动序列化并设置 Content-Type
response = requests.post(url, headers=headers, json=payload)
data = response.json()
print("图片生成参数->>",payload)
print("图片生成->>",data)
return data
def query_result(workflow_id, execute_id,_coze_token=coze_token):
url = f"{base_url}/workflows/{workflow_id}/run_histories/{execute_id}"
headers = {
"Authorization": f"Bearer {_coze_token}",
"Content-Type": "application/json"
}
response = requests.get(url, headers=headers)
data = response.json()
print(f"{execute_id}】结果查询->>",data)
return data
if __name__ == '__main__':
from config import workflow_id
parameters = {
"images": [
{
"file_id": "7612667079497613350",
},{
"file_id": "7612667042990768174",
},{
"file_id": "7612667111332577332",
}
]
}
# resp = workflow_run("7607680760402100258",parameters,_coze_token="sat_ZVNLR9Om54A3iMlJWAasHF9kbtDZnR3BjwjAxRwe8x7igEW446y5ROyVlW1UlpVX")
# print(resp)
# resp = query_result("7607680760402100258","7612668958797447187",_coze_token="sat_ZVNLR9Om54A3iMlJWAasHF9kbtDZnR3BjwjAxRwe8x7igEW446y5ROyVlW1UlpVX")
# print(resp)
resp = upload_file("D:\私单交付\maixiang_AI\测试图片数据\IMG_2686.JPG",_coze_token="sat_ZVNLR9Om54A3iMlJWAasHF9kbtDZnR3BjwjAxRwe8x7igEW446y5ROyVlW1UlpVX")
print(resp)

View File

@@ -1,261 +0,0 @@
"""
生成图片 API - 供 pywebview 前端调用
流程:上传图片 -> workflow_run -> 轮询 query_result -> 解析返回图片 URL
"""
import base64
import json
import os
import tempfile
import time
import re
import traceback
import requests
from coze import upload_file, workflow_run, query_result
from config import workflow_id
from ali_oss import upload_data_urls as oss_upload_data_urls
def _data_url_to_temp_file(data_url: str, allow_video: bool = False) -> str:
"""将 base64 data URL 转为临时文件路径"""
# data:image/png;base64,xxxx 或 data:video/mp4;base64,xxxx
if allow_video:
match = re.match(r'data:(?:image|video)/(\w+);base64,(.+)', data_url)
else:
match = re.match(r'data:image/(\w+);base64,(.+)', data_url)
if not match:
raise ValueError('无效的 data URL 格式')
mime = match.group(1).lower()
ext_map = {'png': 'png', 'webp': 'png', 'jpeg': 'jpg', 'jpg': 'jpg', 'mp4': 'mp4', 'webm': 'webm'}
ext = ext_map.get(mime, 'jpg')
data = base64.b64decode(match.group(2))
fd, path = tempfile.mkstemp(suffix=f'.{ext}')
try:
os.write(fd, data)
finally:
os.close(fd)
return path
def _upload_video(video_data_url: str) -> str:
"""上传视频,返回 file_id"""
if not video_data_url or not isinstance(video_data_url, str):
raise ValueError('无效的视频数据')
path = _data_url_to_temp_file(video_data_url, allow_video=True)
try:
resp = upload_file(path)
if resp.get('code') == 0 and resp.get('data', {}).get('id'):
return resp['data']['id']
raise RuntimeError(f'视频上传失败: {resp.get("msg", resp)}')
finally:
try:
os.unlink(path)
except Exception:
pass
def _upload_images(image_data_urls: list) -> list:
"""上传多张图片,返回 file_id 列表"""
file_ids = []
temp_paths = []
try:
for data_url in (image_data_urls or []):
if not data_url or not isinstance(data_url, str):
continue
if data_url.startswith("http"):
fd, path = tempfile.mkstemp(suffix=f'.png')
data_content = requests.get(data_url).content
os.write(fd, data_content)
else:
path = _data_url_to_temp_file(data_url)
temp_paths.append(path)
resp = upload_file(path)
if resp.get('code') == 0 and resp.get('data', {}).get('id'):
file_ids.append(resp['data']['id'])
else:
raise RuntimeError(f'上传失败: {resp.get("msg", resp)}')
return file_ids
finally:
for p in temp_paths:
try:
os.unlink(p)
except Exception:
pass
def _parse_output(output_str: str) -> list:
"""解析 query_result 中的 output提取图片 URL 列表"""
try:
outer = json.loads(output_str)
inner_str = outer.get('Output', '{}')
inner = json.loads(inner_str)
data_str = inner.get('data', '[]')
data = json.loads(data_str)
urls = data.get("images")
long_image_url = data.get("merged_image_url")
return urls if isinstance(urls, list) else [],long_image_url
except Exception:
return [],None
def _parse_output_text(output_str: str) -> list:
"""解析 query_result 中的 output提取图片 URL 列表"""
try:
outer = json.loads(output_str)
inner_str = outer.get('Output', '{}')
inner = json.loads(inner_str)
data_str = inner.get('data', '')
if isinstance(data_str,str):
data = json.loads(data_str).get("reverse_prompt")
return [data]
pattern = r'【?图像 \d+】.*?(?=【?图像 \d+】|$)'
segments = re.findall(pattern, data_str, re.DOTALL)
return segments
except Exception:
return ""
def _poll_until_done(wf_id: str, execute_id: str, interval: float =10, timeout: int = 600*2, is_text: bool = False) -> list:
"""轮询直到成功或超时,返回图片 URL 列表"""
start = time.time()
while time.time() - start < timeout:
resp = query_result(wf_id, execute_id)
if resp.get('code') != 0:
raise RuntimeError(f'查询失败: {resp.get("msg", resp)}')
items = resp.get('data') or []
if not items:
time.sleep(interval)
continue
item = items[0]
status = item.get('execute_status', '')
if status == 'Success':
output = item.get('output', '{}')
if is_text:
return _parse_output_text(output)
return _parse_output(output)
if status and status not in ('Running', 'Pending', ''):
raise RuntimeError(f'执行失败: {status}')
time.sleep(interval)
raise RuntimeError('生成超时')
def _res_to_2k(res: str) -> str:
"""统一分辨率为 2K/4K"""
r = (res or '2k').strip().upper()
return '4K' if r == '4K' else '2K'
def generate(params: dict) -> dict:
"""
生成图片
params: {
menu: int, # 1=图片反推 2=图片编辑 3=随机海报 4=克隆海报 5=服饰穿搭
prompt: str, # 用户自定义指令 (menu=1 必填)
ref_images: list, # base64 data URLs - 参考图
video: str, # base64 data URL - 视频 (menu=1 可选仅支持1个)
model_images: list, # base64 data URLs - 多模特图 (menu!=1最多5张)
name: str, desc: str, ratio: str, resolution: str, count: int,
language: str, style: str, batch_prompt: list, brand_name: str,
Ingredients: str, activity: str, mode: str, texts: list,
proc_images: list, layout_image: str,
}
返回: { success: bool, urls: list, prompts: list, error: str }
"""
original_urls = []
api_key = (params.get('api_key') or '').strip()
try:
menu = int(params.get('menu', 2))
ref_data = params.get('ref_images') or []
proc_data = params.get('proc_images') or []
layout_data = params.get('layout_image')
video_data = params.get('video')
model_data = params.get('model_images') or []
# menu 1: 反推词,简化参数
if menu == 1:
base_params = {'menu': 1, 'prompt': str(params.get('prompt', '')).strip() or ''}
ref_ids = _upload_images(ref_data) if ref_data else []
if ref_ids:
base_params['ref_images'] = [{'file_id': fid} for fid in ref_ids]
if video_data:
try:
vid = _upload_video(video_data)
base_params['video'] = {'file_id': vid}
except Exception as ve:
return {'success': False, 'urls': [], 'prompts': [], 'error': f'视频上传失败: {ve}'}
base_params = {k: v for k, v in base_params.items() if v is not None and v != ''}
base_params["api_key"] = api_key
resp = workflow_run(workflow_id, base_params)
if resp.get('code') != 0:
return {'success': False, 'urls': [], 'prompts': [], 'error': resp.get('msg', str(resp))}
execute_id = resp.get('execute_id')
if not execute_id:
return {'success': False, 'urls': [], 'prompts': [], 'error': '未返回 execute_id'}
prompts = _poll_until_done(workflow_id, str(execute_id), is_text=True)
# menu 1 返回提示词列表,统一转为字符串
# prompts = [str(x) for x in result_list] if result_list else []
# prompts = [prompts]
return {'success': True, 'urls': [], 'prompts': prompts, 'error': ''}
# menu != 1: 原有逻辑
all_ref = list(ref_data)
if layout_data:
all_ref = [layout_data] + list(ref_data)
all_originals = list(all_ref) + list(proc_data) + list(model_data)
if all_originals:
try:
original_urls = oss_upload_data_urls(all_originals, prefix="originals")
except Exception:
traceback.print_exc()
pass
ref_ids = _upload_images(all_ref) if all_ref else []
proc_ids = _upload_images(proc_data) if proc_data else []
model_ids = _upload_images(model_data) if model_data else []
res = _res_to_2k(params.get('resolution', '2K'))
base_params = {
'name': str(params.get('name', '')).strip(),
'ratio': str(params.get('ratio', '')).strip(),
'menu': menu,
'resolution': res,
'count': int(params.get('count', 1)),
'desc': str(params.get('desc', '')).strip(),
'language': str(params.get('language', '中文')).strip() or '中文',
'style': str(params.get('style', '')).strip(),
'prompt': str(params.get('prompt', '')).strip(),
'batch_prompt': params.get('batch_prompt') or [],
'brand_name': str(params.get('brand_name', '')).strip(),
'Ingredients': str(params.get('Ingredients', '')).strip(),
'activity': str(params.get('activity', '')).strip(),
'mode': str(params.get('mode', '1')).strip() or '1',
'texts': params.get('text') or [],
"api_key" : api_key
}
if ref_ids:
base_params['ref_images'] = [{'file_id': fid} for fid in ref_ids]
if proc_ids:
base_params['proc_images'] = [{'file_id': fid} for fid in proc_ids]
if model_ids:
base_params['model_images'] = [{'file_id': fid} for fid in model_ids]
base_params = {k: v for k, v in base_params.items() if v is not None and v != ''}
resp = workflow_run(workflow_id, base_params)
if resp.get('code') != 0:
return {'success': False, 'urls': [], 'error': resp.get('msg', str(resp))}
execute_id = resp.get('execute_id')
if not execute_id:
return {'success': False, 'urls': [], 'error': '未返回 execute_id'}
urls,long_image_url = _poll_until_done(workflow_id, str(execute_id))
return {'success': True, 'urls': urls, 'original_urls': original_urls, 'error': '',"long_image_url":long_image_url}
except Exception as e:
return {'success': False, 'urls': [], 'prompts': [], 'original_urls': [], 'error': str(e),"long_image_url":""}

View File

@@ -1,68 +0,0 @@
"""
HTML 模板加密/解密模块
使用 AES (Fernet) 对 HTML 资源进行加密存储,运行时解密后渲染
"""
import base64
import os
from cryptography.fernet import Fernet
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
def _get_fernet_key() -> bytes:
"""从环境变量 HTML_ENCRYPT_KEY 派生 Fernet 密钥,若未设置则使用默认开发密钥"""
raw = os.environ.get(
"HTML_ENCRYPT_KEY",
"maixiang_html_encrypt_default_key_change_in_production",
)
# 使用 PBKDF2 派生 32 字节密钥,再转为 Fernet 所需的 base64url
kdf = PBKDF2HMAC(
algorithm=hashes.SHA256(),
length=32,
salt=b"maixiang_html_salt",
iterations=100000,
)
key_bytes = kdf.derive(raw.encode("utf-8"))
return base64.urlsafe_b64encode(key_bytes)
def encrypt(plain_data: bytes) -> bytes:
"""加密原始数据返回密文bytes"""
key = _get_fernet_key()
f = Fernet(key)
return f.encrypt(plain_data)
def decrypt(encrypted_data: bytes) -> bytes:
"""解密数据返回明文bytes"""
key = _get_fernet_key()
f = Fernet(key)
return f.decrypt(encrypted_data)
def encrypt_file(path: str) -> None:
"""就地加密文件:读取 UTF-8 内容,加密后写回同一路径"""
with open(path, "rb") as f:
plain = f.read()
encrypted = encrypt(plain)
with open(path, "wb") as f:
f.write(encrypted)
def decrypt_file(path: str) -> None:
"""就地解密文件:读取密文,解密后以 UTF-8 写回同一路径"""
with open(path, "rb") as f:
encrypted = f.read()
plain = decrypt(encrypted)
with open(path, "wb") as f:
f.write(plain)
def is_encrypted(data: bytes) -> bool:
"""简单启发式Fernet 密文以 b'gAAAAA' 开头base64 编码后)"""
try:
return data[:7] == b"gAAAAAB"
except Exception:
return False

View File

@@ -1,444 +0,0 @@
from config import cache_path,debug,version,JAVA_API_BASE,JSON_TASK_QUEUE,runing_task,runing_shop
import datetime
import json
import sys
import shutil
import os
os.makedirs(cache_path,exist_ok=True)
if not debug:
today = datetime.datetime.now().strftime("%Y_%m_%d")
sys.stdout = open(os.path.join(cache_path,f'{today}.log'), 'a',buffering=1,encoding='utf-8')
sys.stderr = sys.stdout
from amazon.main import TaskMonitor
import webview
import threading
import time
import requests
import subprocess
from amazon.del_brand import kill_process
from app import run_app
from generate_api import generate
with open("version.txt","w",encoding="utf-8") as file:
file.write(json.dumps({"version":version}))
print(f"""
========================================================
版本: {version}
========================================================
""")
# 获取当前脚本所在目录
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
APP_URL = "http://127.0.0.1:5123"
PORT = 5123
def generate_images(params):
"""供前端调用的生成图片接口"""
return generate(params)
class WindowAPI:
"""暴露给 JavaScript 的窗口控制 API"""
def __init__(self, window):
self._window = window
self._is_maximized = False
window.events.maximized += lambda _: setattr(self, '_is_maximized', True)
window.events.restored += lambda _: setattr(self, '_is_maximized', False)
def close(self):
"""关闭窗口,异步执行清理逻辑"""
def cleanup_and_exit():
"""后台清理线程"""
try:
kill_process('v6')
except Exception as e:
print(f"【退出前】关闭紫鸟浏览器进程异常: {str(e)}")
finally:
time.sleep(0.5)
os._exit(0)
print("===============程序退出================")
# 先销毁窗口,让用户看到立即响应
try:
self._window.destroy()
except:
pass
# 在后台线程执行清理,不阻塞
cleanup_thread = threading.Thread(target=cleanup_and_exit, daemon=True)
cleanup_thread.start()
def minimize(self):
self._window.minimize()
def maximize(self):
self._window.maximize()
def toggle_maximize(self):
if self._is_maximized:
self._window.restore()
else:
self._window.maximize()
def save_image(self, url_or_data, filename='image.png'):
"""弹窗选择保存位置并下载图片。url_or_data 可为 http(s) 链接或 data: base64"""
import base64
import re
result = self._window.create_file_dialog(
webview.SAVE_DIALOG,
save_filename=filename,
file_types=('图片文件 (*.png;*.jpg;*.jpeg)', '所有文件 (*.*)')
)
if not result:
return {'success': False, 'error': '用户取消'}
path = result[0] if isinstance(result, (list, tuple)) else result
try:
if url_or_data.startswith('data:'):
match = re.match(r'data:image/\w+;base64,(.+)', url_or_data)
if not match:
print(url_or_data)
print('无效的 data URL')
return {'success': False, 'error': '无效的 data URL'}
data = base64.b64decode(match.group(1))
else:
import requests
resp = requests.get(url_or_data, timeout=30)
resp.raise_for_status()
data = resp.content
with open(path, 'wb') as f:
f.write(data)
return {'success': True, 'path': path}
except Exception as e:
return {'success': False, 'error': str(e)}
def save_image_to_folder(self, url_or_data, dir_path, filename='image.png'):
"""将图片保存到指定文件夹。url_or_data 可为 http(s) 链接或 data: base64"""
import base64
import re
path = os.path.join(dir_path, filename)
try:
if url_or_data.startswith('data:'):
match = re.match(r'data:image/\w+;base64,(.+)', url_or_data)
if not match:
return {'success': False, 'error': '无效的 data URL'}
data = base64.b64decode(match.group(1))
else:
import requests
resp = requests.get(url_or_data, timeout=30)
resp.raise_for_status()
data = resp.content
os.makedirs(dir_path, exist_ok=True)
with open(path, 'wb') as f:
f.write(data)
return {'success': True, 'path': path}
except Exception as e:
return {'success': False, 'error': str(e)}
def select_folder(self):
# 此方法会在 JavaScript 中被调用,返回选择的文件夹路径
result = webview.windows[0].create_file_dialog(
webview.FOLDER_DIALOG, # 指定为文件夹对话框
allow_multiple=False, # 是否允许多选
directory='' # 初始目录
)
# create_file_dialog 返回的是列表(多选时)或 None
return result[0] if result else ''
def select_brand_xlsx_files(self):
"""品牌爬虫:选择 .xlsx 文件,允许多选,返回路径列表"""
result = webview.windows[0].create_file_dialog(
webview.OPEN_DIALOG,
allow_multiple=True,
file_types=('Excel 文件 (*.xlsx)', '所有文件 (*.*)')
)
print("选择的excel 文件",result)
if not result:
return []
return result if isinstance(result, list) or isinstance(result, tuple) else [result]
def select_brand_folder(self):
"""品牌爬虫:选择文件夹,返回文件夹路径(前端再请求后端展开该目录下所有 xlsx"""
result = webview.windows[0].create_file_dialog(
webview.FOLDER_DIALOG,
allow_multiple=False,
directory=''
)
return result[0] if result else ''
def save_file_from_url(self, url, default_filename='download.zip'):
"""弹窗选择保存位置,从 url 下载文件并保存。用于品牌任务结果 zip 等。"""
print("调用到save_file_from_url方法")
if not url or not url.strip():
return {'success': False, 'error': '下载地址为空'}
result = self._window.create_file_dialog(
webview.SAVE_DIALOG,
save_filename=default_filename,
file_types=('ZIP 压缩包 (*.zip)', '所有文件 (*.*)')
)
if not result:
return {'success': False, 'error': '用户取消'}
path = result[0] if isinstance(result, (list, tuple)) else result
try:
import requests
resp = requests.get(url.strip(), timeout=120, stream=True)
resp.raise_for_status()
with open(path, 'wb') as f:
for chunk in resp.iter_content(chunk_size=65536):
if chunk:
f.write(chunk)
return {'success': True, 'path': path}
except Exception as e:
return {'success': False, 'error': str(e)}
def save_file_from_url_new(self, url, default_filename='download.bin'):
"""弹窗选择保存位置,从 url 下载文件并保存。根据文件后缀动态设置保存类型。"""
print("调用到save_file_from_url_new方法")
if not url or not url.strip():
return {'success': False, 'error': '下载地址为空'}
filename = str(default_filename or 'download.bin').strip() or 'download.bin'
ext = os.path.splitext(filename)[1].lower()
# 根据默认文件名后缀动态设置文件类型,避免固定为 zip 造成误导
if ext == '.zip':
file_types = ('ZIP 压缩包 (*.zip)', '所有文件 (*.*)')
elif ext in ('.xlsx', '.xls'):
file_types = ('Excel 文件 (*.xlsx;*.xls)', '所有文件 (*.*)')
elif ext == '.csv':
file_types = ('CSV 文件 (*.csv)', '所有文件 (*.*)')
elif ext == '.txt':
file_types = ('文本文件 (*.txt)', '所有文件 (*.*)')
elif ext == '.json':
file_types = ('JSON 文件 (*.json)', '所有文件 (*.*)')
else:
file_types = ('所有文件 (*.*)',)
result = self._window.create_file_dialog(
webview.SAVE_DIALOG,
save_filename=filename,
file_types=file_types
)
if not result:
return {'success': False, 'error': '用户取消'}
path = result[0] if isinstance(result, (list, tuple)) else result
try:
import requests
resp = requests.get(url.strip(), timeout=120, stream=True)
resp.raise_for_status()
with open(path, 'wb') as f:
for chunk in resp.iter_content(chunk_size=65536):
if chunk:
f.write(chunk)
return {'success': True, 'path': path}
except Exception as e:
return {'success': False, 'error': str(e)}
def save_template_xlsx(self):
"""弹窗选择保存位置,将品牌文档格式模板 xlsx 保存到用户选择的位置。"""
template_name = '品牌文档格式_模板.xlsx'
template_path = os.path.join(BASE_DIR, 'static', template_name)
if not os.path.isfile(template_path):
return {'success': False, 'error': '模板文件不存在'}
result = self._window.create_file_dialog(
webview.SAVE_DIALOG,
save_filename=template_name,
file_types=('Excel 文件 (*.xlsx)', '所有文件 (*.*)')
)
if not result:
return {'success': False, 'error': '用户取消'}
path = result[0] if isinstance(result, (list, tuple)) else result
try:
shutil.copy2(template_path, path)
return {'success': True, 'path': path}
except Exception as e:
return {'success': False, 'error': str(e)}
def save_template_zip(self):
"""弹窗选择保存位置,将品牌文档格式模板 xlsx 保存到用户选择的位置。"""
import shutil
template_name = '模板2-以文件夹方式上传.zip'
template_path = os.path.join(BASE_DIR, 'static', template_name)
if not os.path.isfile(template_path):
return {'success': False, 'error': '模板文件不存在'}
result = self._window.create_file_dialog(
webview.SAVE_DIALOG,
save_filename=template_name,
file_types=('ZIP 文件 (*.zip)', '所有文件 (*.*)')
)
if not result:
return {'success': False, 'error': '用户取消'}
path = result[0] if isinstance(result, (list, tuple)) else result
try:
shutil.copy2(template_path, path)
return {'success': True, 'path': path}
except Exception as e:
return {'success': False, 'error': str(e)}
def upload_file_to_java(self, file_path, relative_path=None):
"""按本地路径读取文件并上传到 Java 后端临时目录。"""
if not file_path or not str(file_path).strip():
return {'success': False, 'error': '文件路径为空'}
normalized_path = os.path.abspath(str(file_path).strip())
if not os.path.isfile(normalized_path):
return {'success': False, 'error': '文件不存在'}
try:
with open(normalized_path, 'rb') as file_obj:
response = requests.post(
f"{JAVA_API_BASE}/api/files/upload",
files={'file': (os.path.basename(normalized_path), file_obj)},
data={'relativePath': str(relative_path).strip()} if relative_path else None,
timeout=120,
)
response.raise_for_status()
payload = response.json()
if not isinstance(payload, dict):
return {'success': False, 'error': 'Java 返回格式错误'}
return payload
except Exception as e:
return {'success': False, 'error': str(e)}
def open_external_url(self, url):
if not url or not str(url).strip():
return {'success': False, 'error': '链接为空'}
target = str(url).strip()
try:
if sys.platform.startswith('win'):
os.startfile(target)
elif sys.platform == 'darwin':
subprocess.Popen(['open', target])
else:
subprocess.Popen(['xdg-open', target])
return {'success': True}
except Exception as e:
return {'success': False, 'error': str(e)}
def enqueue_json(self, data):
"""保存任务到队列"""
try:
print("==============================")
print(datetime.datetime.now().strftime( "%Y-%m-%d %H:%M:%S"),"调用传入",data)
print("==================================")
payload = data
if isinstance(data, str):
payload = json.loads(data)
if not isinstance(payload, (dict, list)):
return {'success': False, 'error': '仅支持 JSON 对象或数组'}
# if payload.get("type") == "delete-brand-run" and payload.get("data").get("taskId") in runing_task:
# return {'success': False, 'error': '已存在正在执行的删除品牌任务'}
# 判断当前店铺是否正在运行中
if payload.get("data",{}).get("shop_name"):
shop_name = payload.get("data",{}).get("shop_name")
if shop_name in runing_shop:
return {'success': False, 'error': f'当前店铺 {shop_name} 正在执行中,请等待店铺完成之后重试'}
if payload.get("data").get("items"):
shop_name = payload["data"]["items"][0]["shopName"]
if shop_name in runing_shop:
return {'success': False, 'error': f'当前店铺 {shop_name} 正在执行中,请等待店铺完成之后重试'}
JSON_TASK_QUEUE.put(payload)
return {'success': True, 'queue_size': JSON_TASK_QUEUE.qsize()}
except Exception as e:
return {'success': False, 'error': str(e)}
def get_detail_del(self,task_id):
"""获取正在执行的删除品牌任务详情"""
task_info = runing_task.get(task_id)
if not task_info:
return {'success': False, 'error': '任务不存在'}
return {'success': True, 'task_info': task_info}
def stop_task(self,task_id):
"""停止正在执行的删除品牌任务"""
task_info = runing_task.get(task_id)
if not task_info:
return {'success': False, 'error': '任务不存在'}
# 这里可以设置一个标志位,实际的删除品牌任务需要定期检查这个标志位来决定是否停止
task_info['stop_requested'] = True
return {'success': True, 'message': '已请求停止任务'}
def start_flask():
run_app(host='127.0.0.1', port=PORT)
def start_task_monitor():
"""启动任务监控器(在独立线程中运行)"""
monitor = TaskMonitor()
try:
print(f"[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] 任务监控线程已启动")
monitor.start()
except Exception:
import traceback
print(f"[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] 任务监控线程异常退出: {traceback.format_exc()}")
def on_window_closing():
"""窗口关闭事件处理器异步执行清理逻辑避免阻塞UI"""
def cleanup_and_exit():
"""后台清理线程"""
try:
kill_process('v6')
except Exception as e:
print(f"【退出前】关闭紫鸟浏览器进程异常: {str(e)}")
finally:
# 给一点时间让清理完成,然后强制退出
time.sleep(0.5)
os._exit(0)
print("=================程序退出=====================")
# 立即在后台线程执行清理,不阻塞窗口关闭
cleanup_thread = threading.Thread(target=cleanup_and_exit, daemon=True)
cleanup_thread.start()
# 立即返回,让窗口快速关闭,用户不会感觉卡顿
return True
def main():
if not os.path.exists(cache_path):
os.makedirs(cache_path,exist_ok=True)
# 启动 Flask 服务
t = threading.Thread(target=start_flask, daemon=True)
t.start()
# 启动任务监控线程
monitor_thread = threading.Thread(target=start_task_monitor, daemon=True)
monitor_thread.start()
# 等待 Flask 启动
time.sleep(2)
window = webview.create_window(
title="数富AI",
url=APP_URL,
width=1400,
height=900,
resizable=True,
min_size=(1200, 700),
background_color="#1a1a1a",
frameless=False,
easy_drag=True
)
# 为窗口关闭事件添加处理器
window.events.closing += on_window_closing
api = WindowAPI(window)
window.expose(api.close, api.minimize, api.maximize, api.toggle_maximize, generate_images, api.save_image, api.save_image_to_folder, api.select_folder, api.select_brand_xlsx_files, api.select_brand_folder,
api.save_file_from_url, api.save_template_xlsx,api.save_template_zip,api.upload_file_to_java,api.open_external_url,api.enqueue_json,api.get_detail_del,api.save_file_from_url_new)
webview.start(
debug=True,
storage_path=cache_path,
private_mode=False
)
if __name__ == "__main__":
main()

View File

@@ -1,102 +0,0 @@
[project]
name = "crawler-desktop-app"
version = "0.1.0"
description = "Desktop crawler app backend and pywebview shell."
requires-python = ">=3.9,<3.10"
dependencies = [
"alibabacloud-oss-v2==1.2.4",
"altgraph==0.17.5",
"annotated-types==0.7.0",
"anyio==4.12.1",
"Authlib==1.6.8",
"blinker==1.9.0",
"bottle==0.13.4",
"certifi==2026.1.4",
"cffi==2.0.0",
"charset-normalizer==3.4.4",
"click==8.1.8",
"clr_loader==0.2.10",
"colorama==0.4.6",
"cozepy==0.20.0",
"crcmod-plus==2.3.1",
"cryptography==41.0.0",
"cssselect==1.3.0",
"DataRecorder==3.6.2",
"distro==1.9.0",
"DownloadKit==2.0.7",
"DrissionPage==4.1.1.2",
"et_xmlfile==2.0.0",
"exceptiongroup==1.3.1",
"filelock==3.19.1",
"Flask==3.1.3",
"flask-cors==6.0.2",
"h11==0.16.0",
"httpcore==1.0.9",
"httpx==0.28.1",
"idna==3.11",
"importlib_metadata==8.7.1",
"itsdangerous==2.2.0",
"Jinja2==3.1.6",
"loguru>=0.7.3",
"lxml==6.0.2",
"MarkupSafe==3.0.3",
"MouseInfo==0.1.3",
"Nuitka==2.8.6",
"numpy==1.25.2",
"openpyxl==3.1.5",
"ordered-set==4.1.0",
"packaging==26.0",
"pandas==2.3.3",
"pefile==2024.8.26",
"pillow==11.3.0",
"proxy_tools==0.1.0",
"psutil==7.2.2",
"PyAutoGUI==0.9.54",
"pycparser==2.23",
"pycryptodome==3.23.0",
"pydantic==2.12.5",
"pydantic_core==2.41.5",
"PyGetWindow==0.0.9",
"pyinstaller==6.19.0",
"pyinstaller-hooks-contrib==2026.3",
"PyMsgBox==2.0.1",
"PyMySQL==1.1.2",
"pyperclip==1.11.0",
"PyQt5==5.15.11",
"PyQt5-Qt5==5.15.2",
"PyQt5_sip==12.17.1",
"PyRect==0.2.0",
"PyScreeze==1.0.1",
"python-dateutil==2.9.0.post0",
"python-dotenv==1.2.1",
"pythonnet==3.0.5",
"pytweening==1.2.0",
"pytz==2026.1.post1",
"pywebview==6.1",
"pywin32-ctypes==0.2.3",
"requests==2.32.5",
"requests-file==3.0.1",
"six==1.17.0",
"tldextract==5.3.0",
"typing-inspection==0.4.2",
"typing_extensions==4.15.0",
"tzdata==2025.3",
"urllib3==2.6.3",
"websocket-client==1.9.0",
"websockets==14.2",
"Werkzeug==3.1.6",
"zipp==3.23.0",
"zstandard==0.25.0",
]
[tool.uv]
package = false
[tool.pytest.ini_options]
pythonpath = ["."]
[dependency-groups]
dev = [
"pytest>=8.4.2",
"ruff>=0.14.0",
]

View File

@@ -1,82 +0,0 @@
alibabacloud-oss-v2==1.2.4
altgraph==0.17.5
annotated-types==0.7.0
anyio==4.12.1
Authlib==1.6.8
blinker==1.9.0
bottle==0.13.4
certifi==2026.1.4
cffi==2.0.0
charset-normalizer==3.4.4
click==8.1.8
clr_loader==0.2.10
colorama==0.4.6
cozepy==0.20.0
crcmod-plus==2.3.1
cryptography==41.0.0
cssselect==1.3.0
DataRecorder==3.6.2
distro==1.9.0
DownloadKit==2.0.7
DrissionPage==4.1.1.2
et_xmlfile==2.0.0
exceptiongroup==1.3.1
filelock==3.19.1
Flask==3.1.3
flask-cors==6.0.2
h11==0.16.0
httpcore==1.0.9
httpx==0.28.1
idna==3.11
importlib_metadata==8.7.1
itsdangerous==2.2.0
Jinja2==3.1.6
lxml==6.0.2
MarkupSafe==3.0.3
MouseInfo==0.1.3
Nuitka==2.8.6
numpy==1.25.2
openpyxl==3.1.5
ordered-set==4.1.0
packaging==26.0
pandas==2.3.3
pefile==2024.8.26
pillow==11.3.0
proxy_tools==0.1.0
psutil==7.2.2
PyAutoGUI==0.9.54
pycparser==2.23
pycryptodome==3.23.0
pydantic==2.12.5
pydantic_core==2.41.5
PyGetWindow==0.0.9
pyinstaller==6.19.0
pyinstaller-hooks-contrib==2026.3
PyMsgBox==2.0.1
PyMySQL==1.1.2
pyperclip==1.11.0
PyQt5==5.15.11
PyQt5-Qt5==5.15.2
PyQt5_sip==12.17.1
PyRect==0.2.0
PyScreeze==1.0.1
python-dateutil==2.9.0.post0
python-dotenv==1.2.1
pythonnet==3.0.5
pytweening==1.2.0
pytz==2026.1.post1
pywebview==6.1
pywin32-ctypes==0.2.3
requests==2.32.5
requests-file==3.0.1
six==1.17.0
tldextract==5.3.0
typing-inspection==0.4.2
typing_extensions==4.15.0
tzdata==2025.3
urllib3==2.6.3
websocket-client==1.9.0
websockets==14.2
Werkzeug==3.1.6
zipp==3.23.0
zstandard==0.25.0

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.9 MiB

View File

View File

@@ -1,411 +0,0 @@
import json
import pytest
from amazon import base
class FakeResponse:
def __init__(self, payload):
self.payload = payload
def json(self):
return self.payload
class FakeWait:
def __init__(self, *, raise_on_doc_loaded=False):
self.doc_loaded_calls = []
self.displayed_calls = []
self.raise_on_doc_loaded = raise_on_doc_loaded
def doc_loaded(self, **kwargs):
if self.raise_on_doc_loaded:
raise RuntimeError("doc load failed")
self.doc_loaded_calls.append(kwargs)
def displayed(self, **kwargs):
self.displayed_calls.append(kwargs)
return True
class FakeElement:
def __init__(self, *, text="", value=None):
self.text = text
self.value = value
self.clicks = 0
self.inputs = []
self.wait = FakeWait()
def click(self, *args, **kwargs):
self.clicks += 1
def input(self, value, clear=False):
self.inputs.append((value, clear))
class FakeTab:
def __init__(self, *, ele_returns=None, eles_returns=None, raise_on_doc_loaded=False):
self.ele_returns = {key: list(values) for key, values in (ele_returns or {}).items()}
self.eles_returns = {key: list(values) for key, values in (eles_returns or {}).items()}
self.wait = FakeWait(raise_on_doc_loaded=raise_on_doc_loaded)
self.refreshes = 0
def ele(self, locator, timeout=None):
values = self.ele_returns.get(locator)
if not values:
return None
return values.pop(0)
def eles(self, locator, timeout=None):
values = self.eles_returns.get(locator)
if not values:
return []
return values.pop(0)
def refresh(self):
self.refreshes += 1
def test_ziniao_driver_initializes_browser_state():
user_info = {
"company": "test-company",
"username": "test-user",
"password": "test-password",
}
driver = base.ZiniaoDriver(user_info, socket_port=19999)
assert driver.user_info is user_info
assert driver.socket_port == 19999
assert driver.client_path is None
assert driver.browser is None
assert driver.tab is None
assert driver.store_id is None
def test_amazon_base_inherits_ziniao_driver():
driver = base.AmamzonBase({})
assert isinstance(driver, base.ZiniaoDriver)
def test_kill_process_uses_ziniao_driver(monkeypatch):
calls = []
class FakeDriver:
def __init__(self, user_info):
calls.append(("init", user_info))
def kill_process(self, version):
calls.append(("kill_process", version))
monkeypatch.setattr(base, "ZiniaoDriver", FakeDriver)
base.kill_process("v6")
assert calls == [("init", {}), ("kill_process", "v6")]
def test_get_zinaio_exe_returns_none_when_winreg_unavailable(monkeypatch):
driver = base.ZiniaoDriver({})
monkeypatch.setattr(base, "winreg", None)
messages = []
sink_id = base.logger.add(messages.append, level="ERROR", format="{message}")
try:
result = driver.get_zinaio_exe()
finally:
base.logger.remove(sink_id)
assert result is None
assert any("winreg" in message for message in messages)
@pytest.mark.parametrize(
("store_info", "expected_key"),
[
("12345", "browserId"),
("oauth-token", "browserOauth"),
],
)
def test_open_store_payload_uses_id_or_oauth(monkeypatch, store_info, expected_key):
payloads = []
def fake_post(url, data, timeout):
payloads.append(json.loads(data.decode("utf-8")))
return FakeResponse({"statusCode": 0, "debuggingPort": 9222})
monkeypatch.setattr(base.requests, "post", fake_post)
result = base.ZiniaoDriver({"company": "c"}).open_store(store_info)
assert result["debuggingPort"] == 9222
assert payloads[0]["action"] == "startBrowser"
assert payloads[0][expected_key] == store_info
@pytest.mark.parametrize(
("payload", "expected"),
[
({"statusCode": 0, "browserList": [{"browserName": "shop"}]}, [{"browserName": "shop"}]),
({"statusCode": -10003}, None),
({"statusCode": 500, "message": "failed"}, None),
],
)
def test_get_browser_list_status_handling(monkeypatch, payload, expected):
monkeypatch.setattr(base.requests, "post", lambda *args, **kwargs: FakeResponse(payload))
assert base.ZiniaoDriver({}).get_browser_list() == expected
def test_update_core_retries_none_then_returns_on_success(monkeypatch):
calls = []
responses = [FakeResponse(None), FakeResponse({"statusCode": 0})]
def fake_post(url, data, timeout):
calls.append(json.loads(data.decode("utf-8")))
return responses.pop(0)
monkeypatch.setattr(base.requests, "post", fake_post)
monkeypatch.setattr(base.time, "sleep", lambda seconds: None)
base.ZiniaoDriver({}).update_core()
assert [payload["action"] for payload in calls] == ["updateCore", "updateCore"]
def test_wait_until_client_ready_polls_until_available(monkeypatch):
readiness = [False, False, True]
sleeps = []
driver = base.ZiniaoDriver({})
monkeypatch.setattr(driver, "_is_client_ready", lambda: readiness.pop(0))
monkeypatch.setattr(base.time, "sleep", lambda seconds: sleeps.append(seconds))
assert driver._wait_until_client_ready(timeout=10) is True
assert sleeps == [base.CLIENT_READY_INTERVAL, base.CLIENT_READY_INTERVAL]
def test_ziniao_webdriver_tail_redacts_password(monkeypatch, tmp_path):
appdata = tmp_path / "AppData" / "Roaming"
log_dir = appdata / "ziniaobrowser" / "instances" / "userdata1" / "logs" / "client"
log_dir.mkdir(parents=True)
log_file = log_dir / "webdriver.20260501.log"
log_file.write_text(
'[error] {"action":"updateCore","username":"u","password":"secret"}\n',
encoding="utf-8",
)
messages = []
monkeypatch.setenv("APPDATA", str(appdata))
monkeypatch.setattr(base.time, "strftime", lambda fmt: "20260501")
sink_id = base.logger.add(messages.append, level="ERROR", format="{message}")
try:
base.ZiniaoDriver({})._log_ziniao_webdriver_tail()
finally:
base.logger.remove(sink_id)
output = "\n".join(messages)
assert "webdriver.20260501.log" in output
assert '"password":"***"' in output
assert "secret" not in output
def test_close_store_success_uses_current_store(monkeypatch):
payloads = []
def fake_post(url, data, timeout):
payloads.append(json.loads(data.decode("utf-8")))
return FakeResponse({"statusCode": 0, "ok": True})
monkeypatch.setattr(base.requests, "post", fake_post)
driver = base.ZiniaoDriver({})
driver.store_id = "oauth-1"
result = driver.close_store()
assert result["ok"] is True
assert payloads[0]["action"] == "stopBrowser"
assert payloads[0]["browserOauth"] == "oauth-1"
def test_close_store_login_failure_raises(monkeypatch):
monkeypatch.setattr(base.requests, "post", lambda *args, **kwargs: FakeResponse({"statusCode": -10003}))
with pytest.raises(RuntimeError, match="【close_store】登录失败"):
base.ZiniaoDriver({}).close_store("oauth-1")
def test_open_shop_returns_message_when_shop_missing(monkeypatch):
driver = base.ZiniaoDriver({})
monkeypatch.setattr(driver, "start_client", lambda: None)
monkeypatch.setattr(driver, "get_browser_list", lambda: [{"browserName": "other"}])
assert driver.open_shop("target") == "店铺不存在"
assert driver.store_id is None
def test_open_shop_opens_matching_store(monkeypatch):
opened_pages = []
fake_browser = object()
driver = base.ZiniaoDriver({})
monkeypatch.setattr(driver, "start_client", lambda: None)
monkeypatch.setattr(driver, "get_browser_list", lambda: [{"browserName": "target", "browserOauth": "oauth-1"}])
monkeypatch.setattr(
driver,
"open_store",
lambda store_id: {
"browserOauth": "oauth-2",
"debuggingPort": 9222,
"ipDetectionPage": "https://ip-check",
"launcherPage": "https://launcher",
},
)
monkeypatch.setattr(driver, "get_browser", lambda port: fake_browser)
monkeypatch.setattr(driver, "open_ip_check", lambda browser, url: True)
monkeypatch.setattr(driver, "open_launcher_page", lambda page, browser: opened_pages.append((page, browser)))
assert driver.open_shop("target") is fake_browser
assert driver.store_id == "oauth-2"
assert opened_pages == [("https://launcher", fake_browser)]
def test_switching_countries_returns_true_when_already_current():
current_country = FakeElement(text="西班牙")
driver = base.AmamzonBase({})
driver.browser = object()
driver.tab = FakeTab(
ele_returns={
'xpath://div[@class="dropdown-account-switcher-header-label"]/span[last()]': [current_country],
}
)
assert driver.switch_to_country("西班牙") is True
def test_switching_countries_returns_false_when_target_missing(monkeypatch):
monkeypatch.setattr(base.time, "sleep", lambda seconds: None)
driver = base.AmamzonBase({})
driver.browser = object()
driver.tab = FakeTab(
ele_returns={
'xpath://div[@class="dropdown-account-switcher-header-label"]/span[last()]': [FakeElement(text="英国")],
'xpath://div[@class="dropdown-account-switcher-header-label"]': [FakeElement()],
'xpath://div[@class="dropdown-account-switcher-list-item"]': [FakeElement()],
'xpath://div[@class="dropdown-account-switcher-list-item dropdown-account-switcher-list-item-indented" and @title="西班牙"]': [
None
],
}
)
assert driver.switch_to_country("西班牙") is False
def test_switching_countries_clicks_and_verifies_target(monkeypatch):
monkeypatch.setattr(base.time, "sleep", lambda seconds: None)
dropdown = FakeElement()
first_item = FakeElement()
target = FakeElement()
driver = base.AmamzonBase({})
driver.browser = object()
driver.tab = FakeTab(
ele_returns={
'xpath://div[@class="dropdown-account-switcher-header-label"]/span[last()]': [
FakeElement(text="英国"),
FakeElement(text="西班牙"),
],
'xpath://div[@class="dropdown-account-switcher-header-label"]': [dropdown],
'xpath://div[@class="dropdown-account-switcher-list-item"]': [first_item],
'xpath://div[@class="dropdown-account-switcher-list-item dropdown-account-switcher-list-item-indented" and @title="西班牙"]': [
target
],
}
)
assert driver.switch_to_country("西班牙") is True
assert dropdown.clicks == 1
assert first_item.clicks == 1
assert target.clicks == 1
def test_open_country_dropdown_waits_for_delayed_country_list(monkeypatch):
monkeypatch.setattr(base.time, "sleep", lambda seconds: None)
dropdown = FakeElement()
first_item = FakeElement()
driver = base.AmamzonBase({})
driver.tab = FakeTab(
ele_returns={
'xpath://div[@class="dropdown-account-switcher-header-label"]': [dropdown],
'xpath://div[@class="dropdown-account-switcher-list-item"]': [None, first_item],
}
)
assert driver._open_country_dropdown() is True
assert dropdown.clicks == 2
assert first_item.clicks == 1
def test_login_returns_true_when_otp_submit_succeeds(monkeypatch):
monkeypatch.setattr(base.time, "sleep", lambda seconds: None)
password_input = FakeElement()
password_submit = FakeElement()
otp_input = FakeElement(value="123456")
otp_submit = FakeElement()
driver = base.AmamzonBase({})
driver.tab = FakeTab(
eles_returns={
'xpath://input[@type="password"]': [[password_input]],
'xpath://input[@id="signInSubmit"]': [[password_submit]],
'xpath://span[@id="auth-send-code" and contains(string(.),"发送一次性密码")]': [[]],
'xpath://input[@name="otpCode"]': [[otp_input]],
'xpath://input[@id="auth-signin-button"]': [[otp_submit]],
'xpath://div[@id="auth-error-message-box"]': [[]],
}
)
assert driver.login("secret") is True
assert password_input.inputs == [("secret", True)]
assert password_submit.clicks == 1
assert otp_submit.clicks == 1
def test_login_refreshes_on_otp_error(monkeypatch):
monkeypatch.setattr(base.time, "sleep", lambda seconds: None)
otp_input = FakeElement(value="123456")
driver = base.AmamzonBase({})
driver.tab = FakeTab(
eles_returns={
'xpath://input[@type="password"]': [[]],
'xpath://input[@id="signInSubmit"]': [[]],
'xpath://span[@id="auth-send-code" and contains(string(.),"发送一次性密码")]': [[]],
'xpath://input[@name="otpCode"]': [[otp_input]],
'xpath://input[@id="auth-signin-button"]': [[FakeElement()]],
'xpath://div[@id="auth-error-message-box"]': [[FakeElement(text="验证码错误")]],
}
)
assert driver.login("secret") is False
assert driver.tab.refreshes == 1
def test_login_returns_false_when_no_otp_input(monkeypatch):
monkeypatch.setattr(base.time, "sleep", lambda seconds: None)
driver = base.AmamzonBase({})
driver.tab = FakeTab(
eles_returns={
'xpath://input[@type="password"]': [[]],
'xpath://input[@id="signInSubmit"]': [[]],
'xpath://span[@id="auth-send-code" and contains(string(.),"发送一次性密码")]': [[]],
'xpath://input[@name="otpCode"]': [[]],
'xpath://input[@id="auth-signin-button"]': [[]],
}
)
assert driver.login("secret") is False
def test_login_returns_false_on_exception():
driver = base.AmamzonBase({})
driver.tab = FakeTab(raise_on_doc_loaded=True)
assert driver.login("secret") is False

File diff suppressed because it is too large Load Diff

View File

@@ -1 +0,0 @@
8cecee3bc02a178bf372ca1c3d02fc5cae3c0a51c8346f6e6a710988599c08be

View File

@@ -1,249 +0,0 @@
import hashlib
import platform
import subprocess
import uuid
import os
from typing import Optional
class DeviceIDGenerator:
"""
Windows设备唯一ID生成器
通过收集多个硬件特征来生成稳定的设备唯一标识符
"""
def __init__(self, use_cache: bool = False, cache_file: str = ".device_id"):
"""
初始化设备ID生成器
Args:
use_cache: 是否使用本地缓存
cache_file: 缓存文件名
"""
self.use_cache = use_cache
self.cache_file = cache_file
def _run_wmic_command(self, command: str) -> Optional[str]:
"""
执行WMIC命令并返回结果
Args:
command: WMIC命令
Returns:
命令执行结果失败则返回None
"""
try:
result = subprocess.run(
command,
shell=True,
capture_output=True,
text=True,
timeout=10
)
if result.returncode == 0 and result.stdout.strip():
return result.stdout.strip()
except (subprocess.TimeoutExpired, Exception):
pass
return None
def _get_motherboard_serial(self) -> Optional[str]:
"""获取主板序列号"""
return self._run_wmic_command("wmic baseboard get serialnumber /value")
def _get_cpu_id(self) -> Optional[str]:
"""获取CPU ID"""
return self._run_wmic_command("wmic cpu get processorid /value")
def _get_bios_serial(self) -> Optional[str]:
"""获取BIOS序列号"""
return self._run_wmic_command("wmic bios get serialnumber /value")
def _get_disk_serial(self) -> Optional[str]:
"""获取系统盘序列号"""
return self._run_wmic_command("wmic diskdrive get serialnumber /value")
def _get_machine_guid(self) -> Optional[str]:
"""获取Windows机器GUID"""
try:
result = subprocess.run(
'reg query "HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Cryptography" /v MachineGuid',
shell=True,
capture_output=True,
text=True,
timeout=10
)
if result.returncode == 0:
for line in result.stdout.split('\n'):
if 'MachineGuid' in line:
return line.split()[-1]
except Exception:
pass
return None
def _extract_value(self, wmic_output: str) -> str:
"""从WMIC输出中提取实际值"""
if not wmic_output:
return ""
lines = wmic_output.split('\n')
for line in lines:
if '=' in line and not line.strip().endswith('='):
return line.split('=', 1)[1].strip()
return ""
def _collect_hardware_info(self) -> dict:
"""
收集硬件信息
Returns:
包含各种硬件信息的字典
"""
hardware_info = {}
# 主板序列号
motherboard = self._get_motherboard_serial()
hardware_info['motherboard'] = self._extract_value(motherboard) if motherboard else ""
# CPU ID
cpu_id = self._get_cpu_id()
hardware_info['cpu'] = self._extract_value(cpu_id) if cpu_id else ""
# BIOS序列号
bios = self._get_bios_serial()
hardware_info['bios'] = self._extract_value(bios) if bios else ""
# 硬盘序列号
disk = self._get_disk_serial()
hardware_info['disk'] = self._extract_value(disk) if disk else ""
# Windows机器GUID
machine_guid = self._get_machine_guid()
hardware_info['machine_guid'] = machine_guid if machine_guid else ""
# 计算机名称
hardware_info['computer_name'] = platform.node()
# MAC地址作为备用
hardware_info['mac_address'] = ':'.join(['{:02x}'.format((uuid.getnode() >> elements) & 0xff)
for elements in range(0, 2*6, 2)][::-1])
return hardware_info
def _generate_device_id(self, hardware_info: dict) -> str:
"""
基于硬件信息生成设备ID
Args:
hardware_info: 硬件信息字典
Returns:
32位十六进制设备ID
"""
# 过滤掉空值,并按键排序确保一致性
filtered_info = {k: v for k, v in hardware_info.items() if v and v.strip()}
# 如果没有任何硬件信息使用MAC地址作为后备方案
if not filtered_info:
filtered_info = {'mac_address': hardware_info.get('mac_address', str(uuid.getnode()))}
# 将所有信息连接成字符串
info_string = '|'.join(f"{k}:{v}" for k, v in sorted(filtered_info.items()))
# 使用SHA256生成哈希值
hash_object = hashlib.sha256(info_string.encode('utf-8'))
device_id = hash_object.hexdigest()
return device_id
def _load_cached_device_id(self) -> Optional[str]:
"""从缓存文件加载设备ID"""
try:
if os.path.exists(self.cache_file):
with open(self.cache_file, 'r', encoding='utf-8') as f:
cached_id = f.read().strip()
if len(cached_id) == 64: # SHA256哈希长度
return cached_id
except Exception:
pass
return None
def _save_device_id_to_cache(self, device_id: str) -> None:
"""将设备ID保存到缓存文件"""
try:
with open(self.cache_file, 'w', encoding='utf-8') as f:
f.write(device_id)
except Exception:
pass
def get_device_id(self) -> str:
"""
获取设备唯一ID
Returns:
64字符的十六进制设备ID
"""
# 如果启用缓存,先尝试从缓存加载
if self.use_cache:
cached_id = self._load_cached_device_id()
if cached_id:
return cached_id
# 收集硬件信息
hardware_info = self._collect_hardware_info()
# 生成设备ID
device_id = self._generate_device_id(hardware_info)
# 保存到缓存
if self.use_cache:
self._save_device_id_to_cache(device_id)
return device_id
def get_device_id_short(self, length: int = 16) -> str:
"""
获取短版本的设备ID
Args:
length: 返回ID的长度
Returns:
指定长度的设备ID
"""
full_id = self.get_device_id()
return full_id[:length]
def get_hardware_info(self) -> dict:
"""
获取硬件信息(用于调试)
Returns:
硬件信息字典
"""
return self._collect_hardware_info()
# 使用示例
def main():
"""使用示例"""
# 创建设备ID生成器实例
device_generator = DeviceIDGenerator()
# 获取完整设备ID64字符
device_id = device_generator.get_device_id()
print(f"完整设备ID: {device_id}")
# 获取短版本设备ID16字符
short_id = device_generator.get_device_id_short(16)
print(f"短设备ID: {short_id}")
# 查看硬件信息(调试用)
hardware_info = device_generator.get_hardware_info()
print("\n硬件信息:")
for key, value in hardware_info.items():
print(f" {key}: {value}")
if __name__ == "__main__":
main()

1501
app/uv.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1 +0,0 @@
{"version": "1.0.56"}

View File

@@ -1,38 +0,0 @@
@echo off
chcp 65001 >nul
setlocal enabledelayedexpansion
echo 正在执行 PowerShell 命令解除文件锁定...
powershell -Command "Get-ChildItem -Path \"%cd%\" -Recurse | Unblock-File"
if %errorlevel% equ 0 (
echo 文件解锁完成
) else (
echo 文件解锁失败,请检查 PowerShell 权限
)
echo.
echo 检查 ShuFuAi.exe.config 文件...
if exist "ShuFuAi.exe.config" (
echo ShuFuAi.exe.config 已存在,跳过创建
) else (
echo 正在创建 ShuFuAi.exe.config...
(
echo ^<?xml version="1.0" encoding="utf-8" ?^>
echo ^<configuration^>
echo ^<runtime^>
echo ^<loadFromRemoteSources enabled="true"/^>
echo ^</runtime^>
echo ^</configuration^>
) > "ShuFuAi.exe.config"
if exist "ShuFuAi.exe.config" (
echo ShuFuAi.exe.config 创建成功
) else (
echo ShuFuAi.exe.config 创建失败,请检查权限
)
)
echo.
echo 操作完成!
pause

View File

@@ -0,0 +1,60 @@
# Multi-Server Start Commands
Use the same packaged jar on both servers. Do not switch code comments before packaging.
Requirements:
- Build once: `mvn clean package -DskipTests`
- Use profile: `server`
- Put JVM options before `-jar`
- Pass `AIIMAGE_INSTANCE_ID` and Redis address from the startup command
- Result file job MQ is expected to stay enabled in packaged `server` deployments
1Panel server 121:
Environment variables:
```env
SPRING_PROFILES_ACTIVE=server
AIIMAGE_INSTANCE_ID=server-121
AIIMAGE_REDIS_HOST=192.168.0.172
AIIMAGE_REDIS_PORT=16379
AIIMAGE_REDIS_PASSWORD=B6COTcY094TYe545
AIIMAGE_REDIS_DATABASE=0
AIIMAGE_SERVER_PORT=18080
AIIMAGE_STORAGE_LOCAL_TEMP_DIR=/app/data/tmp
AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED=true
AIIMAGE_ROCKETMQ_NAME_SERVER=121.196.149.225:9876
AIIMAGE_TRANSIENT_STORAGE_ENDPOINT=http://121.196.149.225:9000
```
Recommended startup command:
```bash
java -Xmx2048M -Xms2048M -jar /app/aiimage-backend-0.0.1-SNAPSHOT.jar
```
Baota server 111:
```bash
/www/server/java/jdk-21.0.2/bin/java -Xmx1024M -Xms256M -jar /app/java/aiimage-backend-0.0.1-SNAPSHOT.jar --spring.profiles.active=server --aiimage.instance-id=server-111 --spring.data.redis.host=47.111.163.154 --spring.data.redis.port=16379 --spring.data.redis.password=B6COTcY094TYe545 --server.port=18080 --aiimage.storage.local-temp-dir=/app/data/tmp
```
Optional Baota server 121 command:
```bash
/www/server/java/jdk-21.0.2/bin/java -Xmx1024M -Xms256M -jar /app/java/aiimage-backend-0.0.1-SNAPSHOT.jar --spring.profiles.active=server --aiimage.instance-id=server-121 --spring.data.redis.host=192.168.0.172 --spring.data.redis.port=16379 --spring.data.redis.password=B6COTcY094TYe545 --server.port=18080 --aiimage.storage.local-temp-dir=/app/data/tmp
```
Successful startup should show a line similar to:
```text
[instance] resolved instanceId=server-121 source=configured
```
or:
```text
[instance] resolved instanceId=server-111 source=configured
```
If the instanceId in the log does not match the startup command, the actual process was not started with the expected command.

View File

@@ -107,7 +107,7 @@ Content-Type: application/json
当前超时配置来自后端配置项:
- `aiimage.delete-brand-progress.patrol-delete-stale-timeout-minutes`
- `aiimage.delete-brand-progress.patrol-delete-initial-timeout-minutes`
- Python should call `POST /api/tasks/{taskId}/heartbeat` every minute; timeout now uses only the normal stale-timeout setting.
## 6. 历史清理

View File

@@ -0,0 +1,79 @@
# Stale 任务收尾约束(公共心跳模块)
> 适用范围:所有使用 [TaskHeartbeatService](../src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatService.java) 的 file_task 模块
> PRODUCT_RISK_RESOLVE / PRICE_TRACK / SHOP_MATCH / PATROL_DELETE / QUERY_ASIN /
> APPEARANCE_PATENT / SIMILAR_ASIN / DELETE_BRAND
## 历史故障2026-06-08
SIMILAR_ASIN taskId=13102 的 file_job 长期持有 RUNNING 等待 Python 凑齐
batchSize 行再发 Coze。Python 端 13:20 异常掉线,但任务到下午 16 点仍卡在 12% 不前进,
30 分钟兜底失效。
链路:
1. [resetStuckJobs](../src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java)
每 60 秒扫一次,把 RUNNING 超 30 分钟的 file_job 改回 PENDING。
2. worker 立刻把它捞起来重跑 `processResultFileJob`
3. 重跑分支命中 "等 Python 上传更多 row" → 调用 `touchJavaSideTaskActivity`
`file_task.updated_at` 刷成 now。
4. `finalizeStaleTasks``lt(updated_at, threshold)` 预过滤,于是这个任务
永远进不到候选集Redis 心跳判定那一步根本走不到。
→ Python 已断线 N 小时,任务仍在循环,永不收尾。
## 约束
### 1. stale-finalize 必须以 Redis 心跳为主信号
`TaskCacheService.touchTaskHeartbeat` 写的 Redis key 仅在 **Python 真实活动**
(心跳接口、上传 result chunk、提交 row时刷新。Java 内部循环不应触碰它。
stale 判定时应:
- 以 Redis heartbeat 为主信号;
- 仅在 Redis 不可用 / 缺失时回落到 `file_task.updated_at`
- 不要再用 `lt(file_task.updated_at, threshold)` 做预过滤
(除非该模块 100% 不存在长期 RUNNING 等待循环)。
参考实现:[SimilarAsinTaskService.finalizeStaleTasks / isHeartbeatStale](../src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java)
和 [AppearancePatentTaskService](../src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java) 中 P1-7 注释段。
### 2. 不要在"等 Python"分支刷 file_task.updated_at
如果模块的 file_job 会持有 RUNNING 状态等待外部上传更多数据,
**这条等待路径上不允许调 `setUpdatedAt(now)` 类方法**
否则会触发 [resetStuckJobs](../src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java)
→ 重跑 → 续命 → 再 reset 的死循环。
如必须刷 `updated_at` 推动其他逻辑,应迁移到独立字段(例如
`file_job.updated_at`、Redis pendingScope 字段),与 stale 判定信号解耦。
### 3. 新增/修改 stale-finalize 流程的 PR 检查清单
- [ ] stale 判定的主信号是 Redis 心跳(不是 `file_task.updated_at`
- [ ] 列出该模块所有刷 `file_task.updated_at` 的代码位置,
其中没有任何一处会被 `processResultFileJob` 在"等 Python"分支重复触发
- [ ] 如果 file_job 会持有长 RUNNINGpull-style 工作流),明确写明 stuck-reset
重跑后的幂等行为(不会续命 updated_at
- [ ] 给 finalize 路径补一个 debug 接口(参考
[DebugTaskRecoveryController](../src/main/java/com/nanri/aiimage/modules/debug/controller/DebugTaskRecoveryController.java)
绕过过滤直接对单 task 强制收尾,便于线上应急
## 当前各模块状态2026-06-08 复核)
| 模块 | stale 入口 | 是否曾出现自我续命 | 是否需要改 |
|---|---|---|---|
| SIMILAR_ASIN | `SimilarAsinTaskService.finalizeStaleTasks` | 是(已修) | 已切 Redis-first |
| APPEARANCE_PATENT | `AppearancePatentTaskService.finalizeStaleTasks` | 是(已修) | 已切 Redis-first |
| PRODUCT_RISK_RESOLVE | `DeleteBrandStaleTaskService.failStaleProductRiskResolveTasks` | 否 | 当前不需要 |
| PRICE_TRACK | `DeleteBrandStaleTaskService.failStalePriceTrackTasks` | 否 | 当前不需要 |
| SHOP_MATCH | `DeleteBrandStaleTaskService.failStaleShopMatchTasks` | 否 | 当前不需要 |
| PATROL_DELETE | `DeleteBrandStaleTaskService.failStalePatrolDeleteTasks` | 否 | 当前不需要 |
| QUERY_ASIN | `DeleteBrandStaleTaskService.failStaleQueryAsinTasks` | 否 | 当前不需要 |
| DELETE_BRAND | `DeleteBrandStaleTaskService.failStaleDeleteBrandTasks` | 否 | 当前不需要 |
| BRAND | `BrandTaskService.failStaleRunningTasks`(独立心跳源) | 否 | 当前不需要 |
下次新增"等 row 凑批" / "pull-style 长 RUNNING file_job"型功能时,
请先读本约束。

View File

@@ -26,6 +26,7 @@
<rocketmq-spring.version>2.3.5</rocketmq-spring.version>
<minio.version>8.5.17</minio.version>
<javassist.version>3.28.0-GA</javassist.version>
<jjwt.version>0.12.6</jjwt.version>
</properties>
<dependencyManagement>
@@ -104,6 +105,23 @@
<artifactId>minio</artifactId>
<version>${minio.version}</version>
</dependency>
<dependency>
<groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt-api</artifactId>
<version>${jjwt.version}</version>
</dependency>
<dependency>
<groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt-impl</artifactId>
<version>${jjwt.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt-jackson</artifactId>
<version>${jjwt.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
@@ -138,6 +156,7 @@
<configuration>
<source>${java.version}</source>
<target>${java.version}</target>
<parameters>true</parameters>
<annotationProcessorPaths>
<path>
<groupId>org.projectlombok</groupId>
@@ -152,6 +171,13 @@
</annotationProcessorPaths>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<argLine>-XX:+EnableDynamicAgentLoading -Xshare:off</argLine>
</configuration>
</plugin>
</plugins>
</build>
</project>

View File

@@ -8,11 +8,21 @@ public class BusinessException extends RuntimeException {
this(null, message);
}
public BusinessException(String message, Throwable cause) {
super(message, cause);
this.code = null;
}
public BusinessException(Integer code, String message) {
super(message);
this.code = code;
}
public BusinessException(Integer code, String message, Throwable cause) {
super(message, cause);
this.code = code;
}
public Integer getCode() {
return code;
}

View File

@@ -1,16 +1,48 @@
package com.nanri.aiimage.common.exception;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.service.TaskOwnerForwardService;
import com.nanri.aiimage.config.TaskOperationLockConfig;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.validation.ConstraintViolationException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.MethodArgumentNotValidException;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.RestControllerAdvice;
import org.springframework.web.context.request.async.AsyncRequestNotUsableException;
import org.springframework.web.server.ResponseStatusException;
import java.io.IOException;
@Slf4j
@RequiredArgsConstructor
@RestControllerAdvice
public class GlobalExceptionHandler {
private final TaskOwnerForwardService taskOwnerForwardService;
@ExceptionHandler(TaskOwnerMismatchException.class)
public Object handleTaskOwnerMismatchException(TaskOwnerMismatchException ex, HttpServletRequest request) {
TaskOperationLockConfig.releaseRequestLock(request);
try {
ResponseEntity<byte[]> response = taskOwnerForwardService.forwardCurrentRequest(ex, request);
return ResponseEntity.status(response.getStatusCode())
.headers(response.getHeaders())
.body(response.getBody());
} catch (BusinessException forwardEx) {
return forwardEx.getCode() == null
? ApiResponse.fail(forwardEx.getMessage())
: ApiResponse.fail(forwardEx.getCode(), forwardEx.getMessage());
} catch (Exception forwardEx) {
log.warn("[instance-routing] forward failed taskId={} operation={} owner={} current={} msg={}",
ex.getTaskId(), ex.getOperation(), ex.getOwnerInstanceId(), ex.getCurrentInstanceId(),
forwardEx.getMessage(), forwardEx);
return ApiResponse.fail(40903, "任务归属实例转发失败: " + forwardEx.getMessage());
}
}
@ExceptionHandler(BusinessException.class)
public ApiResponse<Void> handleBusinessException(BusinessException ex) {
if (Integer.valueOf(40901).equals(ex.getCode())) {
@@ -34,9 +66,39 @@ public class GlobalExceptionHandler {
return ApiResponse.fail(ex.getMessage());
}
@ExceptionHandler(AsyncRequestNotUsableException.class)
public void handleAsyncRequestNotUsableException(AsyncRequestNotUsableException ex) {
log.debug("Client disconnected before response completed: {}", ex.getMessage());
}
@ExceptionHandler(ResponseStatusException.class)
public ResponseEntity<ApiResponse<Void>> handleResponseStatusException(ResponseStatusException ex) {
String message = ex.getReason() == null || ex.getReason().isBlank() ? ex.getMessage() : ex.getReason();
return ResponseEntity.status(ex.getStatusCode()).body(ApiResponse.fail(message));
}
@ExceptionHandler(Exception.class)
public ApiResponse<Void> handleException(Exception ex) {
if (isClientAbort(ex)) {
log.debug("Client disconnected before response completed: {}", ex.getMessage());
return ApiResponse.fail("客户端已断开连接");
}
log.error("Unhandled exception", ex);
return ApiResponse.fail("服务异常: " + ex.getMessage());
}
private boolean isClientAbort(Throwable ex) {
Throwable cursor = ex;
while (cursor != null) {
String className = cursor.getClass().getName();
String message = cursor.getMessage() == null ? "" : cursor.getMessage().toLowerCase();
if (cursor instanceof AsyncRequestNotUsableException
|| cursor instanceof IOException && (message.contains("broken pipe") || message.contains("connection reset"))
|| className.contains("ClientAbortException")) {
return true;
}
cursor = cursor.getCause();
}
return false;
}
}

View File

@@ -0,0 +1,36 @@
package com.nanri.aiimage.common.exception;
public class TaskOwnerMismatchException extends BusinessException {
private final Long taskId;
private final String operation;
private final String ownerInstanceId;
private final String currentInstanceId;
public TaskOwnerMismatchException(Long taskId,
String operation,
String ownerInstanceId,
String currentInstanceId) {
super(40903, "该任务已绑定到另一台服务实例处理,请通过原实例继续处理");
this.taskId = taskId;
this.operation = operation;
this.ownerInstanceId = ownerInstanceId;
this.currentInstanceId = currentInstanceId;
}
public Long getTaskId() {
return taskId;
}
public String getOperation() {
return operation;
}
public String getOwnerInstanceId() {
return ownerInstanceId;
}
public String getCurrentInstanceId() {
return currentInstanceId;
}
}

View File

@@ -0,0 +1,156 @@
package com.nanri.aiimage.common.service;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
import com.nanri.aiimage.config.InstanceRoutingProperties;
import jakarta.servlet.http.HttpServletRequest;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.stereotype.Service;
import org.springframework.util.StreamUtils;
import org.springframework.web.client.RestClient;
import org.springframework.web.util.ContentCachingRequestWrapper;
import java.io.IOException;
import java.util.Enumeration;
import java.util.List;
import java.util.Set;
@Slf4j
@Service
@RequiredArgsConstructor
public class TaskOwnerForwardService {
public static final String FORWARDED_HEADER = "X-AIIMAGE-Owner-Forwarded";
private static final Set<String> HOP_BY_HOP_HEADERS = Set.of(
"connection",
"keep-alive",
"proxy-authenticate",
"proxy-authorization",
"te",
"trailer",
"transfer-encoding",
"upgrade",
"host",
"content-length"
);
private final InstanceRoutingProperties properties;
private volatile RestClient sharedRestClient;
public ResponseEntity<byte[]> forwardCurrentRequest(TaskOwnerMismatchException ex, HttpServletRequest request) {
if (request == null) {
throw new BusinessException(40903, "无法获取当前请求,不能转发到任务归属实例");
}
if (hasAlreadyForwarded(request)) {
throw new BusinessException(40903, "任务归属实例转发检测到循环,请检查实例路由配置");
}
HttpMethod method = HttpMethod.valueOf(request.getMethod());
String url = resolveUrl(ex, currentPathAndQuery(request));
byte[] body = requestBody(request);
HttpHeaders headers = copyForwardHeaders(request, ex.getCurrentInstanceId());
log.info("[instance-routing] forward {} taskId={} operation={} owner={} current={} url={}",
method, ex.getTaskId(), ex.getOperation(), ex.getOwnerInstanceId(), ex.getCurrentInstanceId(), url);
return restClient().method(method)
.uri(url)
.headers(target -> target.addAll(headers))
.body(body)
.retrieve()
.toEntity(byte[].class);
}
private String resolveUrl(TaskOwnerMismatchException ex, String path) {
String owner = ex.getOwnerInstanceId();
String baseUrl = properties.getRoutes().get(owner);
if (baseUrl == null || baseUrl.isBlank()) {
throw new BusinessException(40903, "任务归属实例未配置服务路由:" + owner);
}
return stripTrailingSlash(baseUrl) + ensureLeadingSlash(path);
}
private RestClient restClient() {
RestClient client = sharedRestClient;
if (client != null) {
return client;
}
synchronized (this) {
if (sharedRestClient == null) {
SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory();
requestFactory.setConnectTimeout(properties.getConnectTimeoutMillis());
requestFactory.setReadTimeout(properties.getReadTimeoutMillis());
sharedRestClient = RestClient.builder().requestFactory(requestFactory).build();
}
return sharedRestClient;
}
}
private static boolean hasAlreadyForwarded(HttpServletRequest request) {
String value = request.getHeader(FORWARDED_HEADER);
return value != null && !value.isBlank();
}
private static String currentPathAndQuery(HttpServletRequest request) {
String path = request.getRequestURI();
String query = request.getQueryString();
return query == null || query.isBlank() ? path : path + "?" + query;
}
private static byte[] requestBody(HttpServletRequest request) {
if (request instanceof ContentCachingRequestWrapper wrapper) {
byte[] body = wrapper.getContentAsByteArray();
return body == null ? new byte[0] : body;
}
try {
return StreamUtils.copyToByteArray(request.getInputStream());
} catch (IOException ex) {
throw new BusinessException("读取转发请求体失败:" + ex.getMessage());
}
}
private static HttpHeaders copyForwardHeaders(HttpServletRequest request, String currentInstanceId) {
HttpHeaders headers = new HttpHeaders();
Enumeration<String> headerNames = request.getHeaderNames();
while (headerNames != null && headerNames.hasMoreElements()) {
String headerName = headerNames.nextElement();
if (headerName == null || HOP_BY_HOP_HEADERS.contains(headerName.toLowerCase())) {
continue;
}
Enumeration<String> values = request.getHeaders(headerName);
while (values != null && values.hasMoreElements()) {
headers.add(headerName, values.nextElement());
}
}
headers.set(FORWARDED_HEADER, currentInstanceId == null ? "unknown-instance" : currentInstanceId);
if (!headers.containsKey(HttpHeaders.ACCEPT)) {
headers.setAccept(List.of(MediaType.APPLICATION_JSON));
}
if (!headers.containsKey(HttpHeaders.CONTENT_TYPE)
&& request.getContentType() != null
&& !request.getContentType().isBlank()) {
headers.set(HttpHeaders.CONTENT_TYPE, request.getContentType());
}
return headers;
}
private static String stripTrailingSlash(String value) {
String result = value == null ? "" : value.trim();
while (result.endsWith("/")) {
result = result.substring(0, result.length() - 1);
}
return result;
}
private static String ensureLeadingSlash(String value) {
if (value == null || value.isBlank()) {
return "/";
}
return value.startsWith("/") ? value : "/" + value;
}
}

View File

@@ -0,0 +1,216 @@
package com.nanri.aiimage.common.util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.function.BiConsumer;
import java.util.function.Function;
/**
* Coze 回流数据按 ID 分组传播工具。
*
* <p>业务背景:
* 解析行按 Excel 行顺序排列ID 形如 "1"、"1_1"、"1_2"、"2"、"2_1"。
* 同一 baseId第一个下划线 '_' 之前的部分)的连续行视为同一组——即所谓
* "分组只看下一行数据":按行顺序、相邻同 baseId 行才是同组,遇到不同 baseId 立即开新组。
*
* <p>传播规则:
* <ol>
* <li>单行组(孤立 baseId跳过不做任何处理。</li>
* <li>组内任一行的目标字段值"包含" {@code hitValues} 中某个候选值,
* 则把该组所有行的目标字段统一覆盖为该候选值(标准值)。</li>
* <li>多个候选值同时命中时,按 {@code hitValues} 列表顺序优先匹配
* (列表越靠前优先级越高,比如专利场景把"已侵权"放在"侵权"前面,
* 避免短串先命中导致丢失"已"字)。</li>
* <li>无法解析到对应 result 行resolver 返回 null的 parsed 行不参与
* 组内匹配,但仍占据分组位置——不会打断同 baseId 的连续性,
* 也不会被强行写入。</li>
* </ol>
*
* <p>使用方式:
* <pre>
* // 专利结论列:组内任一行结论命中 "已侵权" 或 "侵权",组内都改为该标准值
* CozeGroupResultPropagator.propagateByGroup(
* receivedRows,
* AppearancePatentParsedRowVo::getDisplayId,
* row -&gt; findResultRow(row, resultMap),
* AppearancePatentResultRowDto::getConclusion,
* AppearancePatentResultRowDto::setConclusion,
* List.of("已侵权", "侵权"),
* "[appearance-patent] taskId=" + task.getId()
* );
* </pre>
*/
public final class CozeGroupResultPropagator {
private static final Logger log = LoggerFactory.getLogger(CozeGroupResultPropagator.class);
/**
* 否定前缀关键字。若当前值同时包含 standard 和这些关键字之一,则不视为命中。
* 用于规避"没有侵权""无侵权""不侵权""未发现明显侵权风险"等被 contains 误判为命中"侵权"的情况。
*/
private static final List<String> NEGATIVE_KEYWORDS = Arrays.asList("没有", "", "", "");
private CozeGroupResultPropagator() {
}
/**
* 按 baseId 连续分组、命中即组内传播。无日志上下文标签的便捷重载。
*/
public static <P, R> int propagateByGroup(List<P> parsedRows,
Function<P, String> displayIdGetter,
Function<P, R> resultRowResolver,
Function<R, String> fieldGetter,
BiConsumer<R, String> fieldSetter,
List<String> hitValues) {
return propagateByGroup(parsedRows, displayIdGetter, resultRowResolver,
fieldGetter, fieldSetter, hitValues, null);
}
/**
* 按 baseId 连续分组、命中即组内传播。
*
* @param parsedRows 解析行(按 Excel 行顺序)
* @param displayIdGetter parsed 行 → ID 字符串(用于计算 baseId
* @param resultRowResolver parsed 行 → 对应的 result DTO找不到返回 null
* @param fieldGetter result DTO → 当前目标字段值
* @param fieldSetter result DTO ← 新目标字段值
* @param hitValues 需要传播的标准值列表,靠前优先匹配(包含语义)
* @param logTag 日志前缀(如 "[appearance-patent] taskId=9975"null 则只在汇总层打日志
* @param <P> parsed 行类型
* @param <R> result DTO 类型
* @return 实际被改写的 result 行数(用于上层日志埋点)
*/
public static <P, R> int propagateByGroup(List<P> parsedRows,
Function<P, String> displayIdGetter,
Function<P, R> resultRowResolver,
Function<R, String> fieldGetter,
BiConsumer<R, String> fieldSetter,
List<String> hitValues,
String logTag) {
if (parsedRows == null || parsedRows.isEmpty()
|| displayIdGetter == null || resultRowResolver == null
|| fieldGetter == null || fieldSetter == null
|| hitValues == null || hitValues.isEmpty()) {
return 0;
}
String tagPrefix = (logTag == null || logTag.isEmpty()) ? "" : logTag + " ";
int totalUpdated = 0;
int n = parsedRows.size();
int i = 0;
while (i < n) {
String currentBaseId = baseId(displayIdGetter.apply(parsedRows.get(i)));
int groupStart = i;
int j = i + 1;
// 连续同 baseId 视为一组
while (j < n && currentBaseId.equals(baseId(displayIdGetter.apply(parsedRows.get(j))))) {
j++;
}
int groupEnd = j; // 半开区间 [groupStart, groupEnd)
i = j;
// 单行组跳过
if (groupEnd - groupStart < 2) {
continue;
}
// 收集组内所有能解析到 result DTO 的行,同时记录显示 ID 方便日志
List<R> groupResults = new ArrayList<>(groupEnd - groupStart);
List<String> groupDisplayIds = new ArrayList<>(groupEnd - groupStart);
for (int k = groupStart; k < groupEnd; k++) {
P parsedRow = parsedRows.get(k);
R resultRow = resultRowResolver.apply(parsedRow);
if (resultRow != null) {
groupResults.add(resultRow);
groupDisplayIds.add(displayIdGetter.apply(parsedRow));
}
}
if (groupResults.isEmpty()) {
continue;
}
// 按 hitValues 顺序检测组内是否有命中(包含语义),命中即定下"标准值"。
// 命中需同时满足:当前值包含 standard 且不含任何否定关键字("没有"/"无"/"不"/"未"
// 避免"没有侵权""无侵权""不侵权""未发现明显侵权风险"这类被 contains 误判为命中。
String matchedStandardValue = null;
outer:
for (String standard : hitValues) {
if (standard == null || standard.isEmpty()) {
continue;
}
for (int k = 0; k < groupResults.size(); k++) {
R row = groupResults.get(k);
String currentValue = fieldGetter.apply(row);
if (currentValue == null || !currentValue.contains(standard)) {
continue;
}
if (containsNegative(currentValue)) {
log.debug("{}group-propagate skip negative baseId={} displayId={} currentValue={} candidate={}",
tagPrefix, currentBaseId, groupDisplayIds.get(k), currentValue, standard);
continue;
}
matchedStandardValue = standard;
break outer;
}
}
if (matchedStandardValue == null) {
continue;
}
// 把组内所有 result DTO 的目标字段统一覆盖为标准值,并逐行打日志
int groupUpdated = 0;
for (int idx = 0; idx < groupResults.size(); idx++) {
R row = groupResults.get(idx);
String currentValue = fieldGetter.apply(row);
if (matchedStandardValue.equals(currentValue)) {
continue;
}
fieldSetter.accept(row, matchedStandardValue);
groupUpdated++;
totalUpdated++;
log.info("{}group-propagate row baseId={} displayId={} oldValue={} newValue={}",
tagPrefix, currentBaseId, groupDisplayIds.get(idx),
currentValue, matchedStandardValue);
}
if (groupUpdated > 0) {
log.info("{}group-propagate group hit baseId={} groupSize={} standardValue={} updatedRows={}",
tagPrefix, currentBaseId, groupResults.size(),
matchedStandardValue, groupUpdated);
}
}
return totalUpdated;
}
/**
* 判断当前值是否含任意否定关键字。命中关键字即视为"未发生"语义,
* 上层应跳过该值,避免被 {@link String#contains(CharSequence)} 误判命中标准值。
*/
private static boolean containsNegative(String value) {
if (value == null || value.isEmpty()) {
return false;
}
for (String neg : NEGATIVE_KEYWORDS) {
if (value.contains(neg)) {
return true;
}
}
return false;
}
/**
* 取 ID 中第一个 '_' 之前的部分。例如 "1" → "1""1_2" → "1"null → ""。
* 与两个 TaskService 内的 {@code baseId} 实现保持一致。
*/
private static String baseId(String id) {
if (id == null) {
return "";
}
String s = id.trim();
int idx = s.indexOf('_');
return idx > 0 ? s.substring(0, idx) : s;
}
}

View File

@@ -0,0 +1,114 @@
package com.nanri.aiimage.common.util;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Predicate;
public final class FailedStatusRowFilter {
private static final String STATUS_HEADER_CN = "状态";
private static final String FAILED_STATUS_ERROR = "错误";
private static final String FAILED_STATUS_FAILED_CN = "失败";
private static final String FAILED_STATUS_FAILED_EN = "failed";
private static final String NO_MATCHED_ROWS_MESSAGE =
"状态列过滤后未找到 错误/失败/FAILED 数据";
private static final Set<String> STATUS_HEADER_ALIASES = Set.of(
normalizeHeader(STATUS_HEADER_CN),
normalizeHeader("status")
);
private static final Set<String> FAILED_STATUS_VALUES = Set.of(
normalizeValue(FAILED_STATUS_ERROR),
normalizeValue(FAILED_STATUS_FAILED_CN),
normalizeValue(FAILED_STATUS_FAILED_EN)
);
private FailedStatusRowFilter() {
}
public static int findStatusColumnIndex(List<String> headers) {
if (headers == null || headers.isEmpty()) {
return -1;
}
for (int i = 0; i < headers.size(); i++) {
if (STATUS_HEADER_ALIASES.contains(normalizeHeader(headers.get(i)))) {
return i;
}
}
return -1;
}
public static boolean matchesFailedStatus(String value) {
return FAILED_STATUS_VALUES.contains(normalizeValue(value));
}
public static boolean isBlankStatus(String value) {
return normalizeValue(value).isBlank();
}
public static String noMatchedRowsMessage() {
return NO_MATCHED_ROWS_MESSAGE;
}
/**
* 暴露表头归一化逻辑给其它模块复用,避免每个模块各自维护一份规则。
* 处理:去 BOM、全角空格转半角、首尾 trim、连续空白合并、转小写
* 并剔除常见分隔符(含中文全角括号、方括号、冒号)。
*/
public static String canonicalizeHeader(String value) {
return normalizeHeader(value);
}
public static <T> FilterResult<T> retainFailedRows(List<T> rows,
boolean statusFilteringEnabled,
Function<T, String> statusReader) {
return retainRows(rows, statusFilteringEnabled, statusReader, FailedStatusRowFilter::matchesFailedStatus);
}
public static <T> FilterResult<T> retainRows(List<T> rows,
boolean statusFilteringEnabled,
Function<T, String> statusReader,
Predicate<String> statusMatcher) {
List<T> safeRows = rows == null ? List.of() : rows;
if (!statusFilteringEnabled) {
return new FilterResult<>(false, safeRows, 0);
}
List<T> filteredRows = new ArrayList<>();
int filteredCount = 0;
for (T row : safeRows) {
String status = statusReader == null ? "" : statusReader.apply(row);
boolean matched = statusMatcher != null && statusMatcher.test(status);
if (matched) {
filteredRows.add(row);
} else {
filteredCount++;
}
}
return new FilterResult<>(true, filteredRows, filteredCount);
}
private static String normalizeHeader(String value) {
// 兼容中文全角括号()、中文方括号【】、中文冒号:,
// 避免“状态(结果)”“标题维度(商标)”等表头识别失败。
return normalizeValue(value).replaceAll("[\\s_\\-()\\[\\]{}::()【】]+", "");
}
private static String normalizeValue(String value) {
if (value == null) {
return "";
}
return value
.replace(String.valueOf((char) 0xFEFF), "")
.replace((char) 0x3000, ' ')
.trim()
.replaceAll("\\s+", " ")
.toLowerCase(Locale.ROOT);
}
public record FilterResult<T>(boolean filterApplied, List<T> rows, int filteredCount) {
}
}

View File

@@ -3,6 +3,9 @@ package com.nanri.aiimage.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import java.util.ArrayList;
import java.util.List;
@Data
@ConfigurationProperties(prefix = "aiimage.appearance-patent")
public class AppearancePatentProperties {
@@ -11,11 +14,25 @@ public class AppearancePatentProperties {
private String cozeWorkflowHistoryPath = "/v1/workflows/{workflow_id}/run_histories/{execute_id}";
private String cozeWorkflowId = "7632683471312355338";
private String cozeToken = "";
private int cozeBatchSize = 10;
private List<CozeCredential> cozeCredentials = new ArrayList<>();
private int cozeCredentialStripeSize = 5;
private int cozeBatchSize = 50;
private int cozeConnectTimeoutMillis = 10000;
private int cozeReadTimeoutMillis = 60000;
private int cozePollIntervalMillis = 5000;
private int cozePollIntervalMillis = 30000;
private int cozePollTimeoutMillis = 600000;
private int staleTimeoutMinutes = 20;
private int staleTimeoutMinutes = 30;
private String staleFinalizeCron = "0 */2 * * * *";
/**
* 末尾不足一批的数据等待该时长后强制提交 Coze。
* Python 回传超时后会直接封口;这个配置处理 Python 慢回传但仍未超时的零头批次。
*/
private int cozeFlushPendingMinutes = 1;
@Data
public static class CozeCredential {
private String name;
private String workflowId;
private String token;
}
}

View File

@@ -0,0 +1,15 @@
package com.nanri.aiimage.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
@Data
@ConfigurationProperties(prefix = "aiimage.brand-check")
public class BrandCheckProperties {
private String baseUrl = "http://47.110.241.161:16890";
private String path = "/brand_check";
private String token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9";
private String defaultStrategy = "Terms";
private int connectTimeoutMillis = 10000;
private int readTimeoutMillis = 60000;
}

View File

@@ -7,7 +7,6 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties(prefix = "aiimage.delete-brand-progress")
public class DeleteBrandProgressProperties {
private long heartbeatTimeoutMinutes = 15;
private long deleteBrandInitialTimeoutMinutes = 20;
private String staleCheckCron = "*/30 * * * * *";
/**
@@ -19,24 +18,30 @@ public class DeleteBrandProgressProperties {
* 商品风险PRODUCT_RISK_RESOLVERUNNING 超时自动收尾/失败,按分钟计算;
* 与删除品牌共用同一定时调度。
*/
private long productRiskStaleTimeoutMinutes = 20;
private long productRiskInitialTimeoutMinutes = 20;
private long productRiskStaleTimeoutMinutes = 30;
/**
* Price track RUNNING timeout auto-finalize/fail threshold in minutes.
*/
private long priceTrackStaleTimeoutMinutes = 20;
private long priceTrackInitialTimeoutMinutes = 20;
private long priceTrackStaleTimeoutMinutes = 30;
/**
* Shop match RUNNING timeout auto-finalize/fail threshold in minutes.
*/
private long shopMatchStaleTimeoutMinutes = 20;
private long shopMatchInitialTimeoutMinutes = 20;
private long shopMatchStaleTimeoutMinutes = 30;
/**
* Patrol delete RUNNING timeout auto-finalize/fail threshold in minutes.
*/
private long patrolDeleteStaleTimeoutMinutes = 20;
private long patrolDeleteInitialTimeoutMinutes = 20;
private long patrolDeleteStaleTimeoutMinutes = 30;
/**
* Query ASIN RUNNING timeout auto-finalize/fail threshold in minutes.
*/
private long queryAsinStaleTimeoutMinutes = 30;
/**
* Withdraw RUNNING timeout auto-finalize/fail threshold in minutes.
*/
private long withdrawStaleTimeoutMinutes = 30;
}

View File

@@ -0,0 +1,24 @@
package com.nanri.aiimage.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
@Data
@ConfigurationProperties(prefix = "aiimage.image-video")
public class ImageVideoProperties {
private String cozeBaseUrl = "https://api.coze.cn";
private String cozeToken = "";
private String cozeWorkflowPath = "/v1/workflow/run";
private String cozeWorkflowHistoryPath = "/v1/workflows/{workflow_id}/run_histories/{execute_id}";
private String douyinCopyWorkflowId = "7652941112982798388";
private String imageVideoWorkflowId = "7659086169628377124";
private String voiceDeleteWorkflowId = "7652954386453299243";
private String voiceListWorkflowId = "7652954332346089498";
private String voiceCloneWorkflowId = "7652954356887961641";
private String voiceSynthesisWorkflowId = "7652954297530105894";
private int cozeConnectTimeoutMillis = 10000;
private int cozeReadTimeoutMillis = 3600000;
private int archiveConnectTimeoutMillis = 10000;
private int archiveReadTimeoutMillis = 600000;
private int archiveMaxAttempts = 3;
}

View File

@@ -1,21 +1,38 @@
package com.nanri.aiimage.config;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.net.InetAddress;
@Slf4j
@Component
public class InstanceMetadata {
private final String instanceId;
private final String hostname;
private final String source;
private final boolean stable;
public InstanceMetadata(@Value("${aiimage.instance-id:}") String configuredInstanceId) {
this.hostname = resolveHostname();
this.instanceId = configuredInstanceId == null || configuredInstanceId.isBlank()
? this.hostname
: configuredInstanceId.trim();
String normalizedConfiguredInstanceId = configuredInstanceId == null ? "" : configuredInstanceId.trim();
if (!normalizedConfiguredInstanceId.isBlank()) {
this.instanceId = normalizedConfiguredInstanceId;
this.source = "configured";
this.stable = true;
log.info("[instance] resolved instanceId={} source={} hostname={} continuityRisk=LOW",
this.instanceId, this.source, this.hostname);
} else {
this.instanceId = this.hostname;
this.source = detectFallbackSource(this.hostname);
this.stable = false;
log.warn("[instance] aiimage.instance-id is not configured, falling back to {}={} as instanceId. "
+ "This may change after restart/redeploy and break task owner continuity. "
+ "Set AIIMAGE_INSTANCE_ID in 1Panel/Docker env, or set -Daiimage.instance-id=<stable-id> in IDEA/local startup.",
this.source, this.hostname);
}
}
public String getInstanceId() {
@@ -26,6 +43,14 @@ public class InstanceMetadata {
return hostname;
}
public String getSource() {
return source;
}
public boolean isStable() {
return stable;
}
private static String resolveHostname() {
try {
return InetAddress.getLocalHost().getHostName();
@@ -34,4 +59,12 @@ public class InstanceMetadata {
return (envHostname == null || envHostname.isBlank()) ? "unknown-host" : envHostname;
}
}
private static String detectFallbackSource(String hostname) {
String envHostname = System.getenv("HOSTNAME");
if (envHostname != null && !envHostname.isBlank() && envHostname.equalsIgnoreCase(hostname)) {
return "env.HOSTNAME";
}
return "os.hostname";
}
}

View File

@@ -0,0 +1,17 @@
package com.nanri.aiimage.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import java.util.LinkedHashMap;
import java.util.Map;
@Data
@ConfigurationProperties(prefix = "aiimage.instance-routing")
public class InstanceRoutingProperties {
private Map<String, String> routes = new LinkedHashMap<>();
private int connectTimeoutMillis = 3000;
private int readTimeoutMillis = 300000;
private int requestBodyCacheLimitBytes = 104857600;
}

View File

@@ -11,5 +11,6 @@ import java.util.List;
public class ModuleCleanupProperties {
private boolean enabled = true;
private String cron = "0 0 0 * * *";
private List<String> moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN"));
private long retentionDays = 7;
private List<String> moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "WITHDRAW", "APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA"));
}

View File

@@ -8,7 +8,9 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
public class OssProperties {
private String region;
private String endpoint;
private String publicEndpoint;
private String bucket;
private String imageVideoBucket;
private String accessKeyId;
private String accessKeySecret;
}

View File

@@ -4,6 +4,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.context.annotation.Configuration;
@Configuration
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class})
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, BrandCheckProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, TaskImageCacheCleanupProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class, ImageVideoProperties.class, InstanceRoutingProperties.class})
public class PropertiesConfig {
}

View File

@@ -6,8 +6,10 @@ import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.web.filter.OncePerRequestFilter;
import org.springframework.web.util.ContentCachingRequestWrapper;
import java.io.IOException;
import java.util.Optional;
@@ -18,54 +20,76 @@ public class RequestTraceFilter extends OncePerRequestFilter {
private static final Logger log = LoggerFactory.getLogger(RequestTraceFilter.class);
private final InstanceMetadata instanceMetadata;
private final int requestBodyCacheLimitBytes;
public RequestTraceFilter(InstanceMetadata instanceMetadata) {
public RequestTraceFilter(InstanceMetadata instanceMetadata,
@Value("${aiimage.instance-routing.request-body-cache-limit-bytes:104857600}") int requestBodyCacheLimitBytes) {
this.instanceMetadata = instanceMetadata;
this.requestBodyCacheLimitBytes = Math.max(1024 * 1024, requestBodyCacheLimitBytes);
}
@Override
protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain)
throws ServletException, IOException {
long start = System.currentTimeMillis();
HttpServletRequest requestToUse = wrapRequestIfNeeded(request, requestBodyCacheLimitBytes);
String remoteAddr = firstNonBlank(
request.getHeader("X-Forwarded-For"),
request.getHeader("X-Real-IP"),
request.getRemoteAddr()
requestToUse.getHeader("X-Forwarded-For"),
requestToUse.getHeader("X-Real-IP"),
requestToUse.getRemoteAddr()
);
String forwardedProto = request.getHeader("X-Forwarded-Proto");
String forwardedHost = firstNonBlank(request.getHeader("X-Forwarded-Host"), request.getHeader("Host"));
String forwardedPort = request.getHeader("X-Forwarded-Port");
String forwardedProto = requestToUse.getHeader("X-Forwarded-Proto");
String forwardedHost = firstNonBlank(requestToUse.getHeader("X-Forwarded-Host"), requestToUse.getHeader("Host"));
String forwardedPort = requestToUse.getHeader("X-Forwarded-Port");
String requestId = firstNonBlank(
request.getHeader("X-Request-Id"),
request.getHeader("X-Amzn-Trace-Id"),
request.getHeader("Traceparent")
requestToUse.getHeader("X-Request-Id"),
requestToUse.getHeader("X-Amzn-Trace-Id"),
requestToUse.getHeader("Traceparent")
);
response.setHeader("X-AIIMAGE-Instance", instanceMetadata.getInstanceId());
response.setHeader("X-AIIMAGE-Host", instanceMetadata.getHostname());
response.setHeader("X-AIIMAGE-Instance-Source", instanceMetadata.getSource());
response.setHeader("X-AIIMAGE-Instance-Stable", String.valueOf(instanceMetadata.isStable()));
try {
filterChain.doFilter(request, response);
filterChain.doFilter(requestToUse, response);
} finally {
long costMs = System.currentTimeMillis() - start;
log.info(
"request-trace instance={} host={} method={} uri={} status={} remote={} forwardedHost={} forwardedProto={} forwardedPort={} requestId={} userAgent={} costMs={}",
"request-trace instance={} source={} stable={} host={} method={} uri={} status={} remote={} forwardedHost={} forwardedProto={} forwardedPort={} requestId={} userAgent={} costMs={}",
instanceMetadata.getInstanceId(),
instanceMetadata.getSource(),
instanceMetadata.isStable(),
instanceMetadata.getHostname(),
request.getMethod(),
request.getRequestURI(),
requestToUse.getMethod(),
requestToUse.getRequestURI(),
response.getStatus(),
remoteAddr,
blankToDash(forwardedHost),
blankToDash(forwardedProto),
blankToDash(forwardedPort),
blankToDash(requestId),
blankToDash(request.getHeader("User-Agent")),
blankToDash(requestToUse.getHeader("User-Agent")),
costMs
);
}
}
private static HttpServletRequest wrapRequestIfNeeded(HttpServletRequest request, int requestBodyCacheLimitBytes) {
if (request instanceof ContentCachingRequestWrapper) {
return request;
}
String method = request.getMethod();
if ("POST".equalsIgnoreCase(method)
|| "PUT".equalsIgnoreCase(method)
|| "PATCH".equalsIgnoreCase(method)
|| "DELETE".equalsIgnoreCase(method)) {
return new ContentCachingRequestWrapper(request, requestBodyCacheLimitBytes);
}
return request;
}
private static String firstNonBlank(String... values) {
for (String value : values) {
if (value != null && !value.isBlank()) {

View File

@@ -31,7 +31,7 @@ public class SecurityConfig {
configuration.setAllowedMethods(List.of("GET", "POST", "PUT", "DELETE", "OPTIONS"));
configuration.setAllowedHeaders(List.of("*"));
configuration.setExposedHeaders(List.of("Content-Disposition"));
configuration.setAllowCredentials(false);
configuration.setAllowCredentials(true);
configuration.setMaxAge(3600L);
UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource();

View File

@@ -3,19 +3,134 @@ package com.nanri.aiimage.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import java.util.ArrayList;
import java.util.List;
@Data
@ConfigurationProperties(prefix = "aiimage.similar-asin")
public class SimilarAsinProperties {
private String cozeBaseUrl = "https://api.coze.cn";
private String cozeWorkflowPath = "/v1/workflow/run";
private String cozeWorkflowHistoryPath = "/v1/workflows/{workflow_id}/run_histories/{execute_id}";
private String cozeWorkflowId = "7632683471312355338";
private String cozeWorkflowId = "7635328462404583478";
private String cozeToken = "";
private int cozeBatchSize = 10;
private List<CozeCredential> cozeCredentials = new ArrayList<>();
private int cozeCredentialStripeSize = 0;
/**
* P0-1单次提交 Coze 工作流的 row 数量。
* 历史值 10在含 puzzle 多图行的场景下频繁触发 720712008
* "node executed out of limit: 1000"。降到 3 以避免节点上限被打爆。
* 出现持续 720712008 时还会被 P1-1 滑窗自适应再降到 1。
* 不影响 AppearancePatentProperties 的同名值。
*/
private int cozeBatchSize = 3;
/**
* img_switch=false 时单次提交 Coze 的 row 数。
* 不走图片检测时工作流压力小,恢复到 10 行一批以提高吞吐;开启图片检测时仍使用 cozeBatchSize。
*/
private int cozeTextOnlyBatchSize = 10;
private int cozeConnectTimeoutMillis = 10000;
private int cozeReadTimeoutMillis = 60000;
private int cozePollIntervalMillis = 5000;
private int cozePollTimeoutMillis = 600000;
private int staleTimeoutMinutes = 20;
private int cozePollIntervalMillis = 30000;
private int cozePollTimeoutMillis = 1800000;
private int staleTimeoutMinutes = 30;
private String staleFinalizeCron = "0 */2 * * * *";
/**
* 同一 credential 两次提交之间的最小间隔(毫秒)。
* 历史值硬编码 30000持锁 sleep导致单凭证仅 2 batch/分钟。
* 几千行任务场景下成为提交吞吐瓶颈,下调到 5000ms 并改为锁外冷却。
* 出现 Coze 限流加重时可通过 AIIMAGE_SIMILAR_ASIN_COZE_SUBMIT_MIN_INTERVAL_MILLIS 调高。
*/
private long cozeSubmitMinIntervalMillis = 5000L;
/**
* 末尾零头 batch 的强制 flush 阈值(分钟):当不足 cozeBatchSize 的零头 row
* 长时间挂着Python 慢回传)时触发提交。
* 任务级实测345 行 / 4h 总耗时中,约 2-3 小时是 batch 永远凑不满 batchSize 在等下一波回传,
* 把阈值从 15 调到 1最多 60s 后 1-2 行也强制提交,让 Coze 提交侧持续进票,
* 总耗时降到与 Python 回传节奏接近。配合 cozeBatchSize=3、cozeSubmitMinIntervalMillis=5000
* 实际不会触发 Coze 限流。出现限流加重再调回 5/10。
*/
private int cozeFlushPendingMinutes = 1;
/**
* 同 batch retry + split retry 共享的最大重试次数。原硬编码 5。
*/
private int cozeSubmitMaxRetryCount = 5;
/**
* 图片嵌入下载线程池大小。原 SimilarAsinImageEmbedder.DOWNLOAD_POOL_SIZE = 8。
* P2-101000+ 行 ×3 列图片场景下pool=16 仍是 assemble 阶段瓶颈(实测下载 244s/918s
* 提到 32 配合 retry=2、global deadline 显著拉低尾延迟;
* 受 2GB 堆约束,单图缩略图维持 300KB 以内,整体内存峰值 ≈ 32 * 300KB ≈ 10MB。
*/
private int imageDownloadPoolSize = 32;
/**
* 单张图片下载超时(秒)。
* P2-10放宽到 8 + retry=1 在快源aiproxy/m.media-amazon下没问题
* 但慢源cbu01.alicdn会一直挂 8s 才进入 retry整体串行时间放大。
* 调到 5s + retry=2让慢源更早重试新连接单图最坏耗时 ≈ 5s * (1+2) = 15s。
*/
private int imageDownloadTimeoutSeconds = 5;
/**
* assemble 阶段 taskImageCache 的字节上限。
* 默认 256MB5000 行 × 3 列 × 平均 100KB = 1.5GB 远超 2GB 堆,
* 用 BoundedImageCache 按字节累计 LRU 淘汰避免爆堆。
* 由于 embed() 写完即 remove(),活跃图片字节通常 ≤ 100MB仅在极端 prefetch 领先场景才会触发淘汰。
* 出现淘汰过频影响命中率时可上调到 512MB2GB 堆约束下不建议超过 768MB。
*/
private long imageCacheMaxBytes = 256L * 1024L * 1024L;
private boolean imageDbCacheEnabled = false;
/**
* 是否在 Coze 请求 parameters 中附带 api_key 字段。
* 默认 true线上 Coze 工作流将该字段视为必填,缺失会得到 4000
* "Missing required parameters";前端传入的 api_key 必须透传到 coze。
* 仅在工作流明确不再需要 api_key 时,可通过环境变量
* AIIMAGE_SIMILAR_ASIN_COZE_INCLUDE_LEGACY_API_KEY=false 关闭。
*/
private boolean cozeIncludeLegacyApiKey = true;
/**
* 是否使用旧的 item 字段顺序 {asin, sku, url, target_urls, title}。
* 默认 false当前实现使用 {asin, url, target_urls, title, sku}。
* 出现兼容问题时可通过 AIIMAGE_SIMILAR_ASIN_COZE_USE_LEGACY_ITEM_ORDER=true
* 切回旧顺序进行回归对比。
*/
private boolean cozeUseLegacyItemFieldOrder = false;
/**
* 是否启用 P0-3 merge 增量缓冲:每个 batch DONE 时仅缓冲 cozeRows
* 不立即合并到 chunkfinalize 前一次性按 chunkScopeHash 分组合并,
* 把 OSS chunk 读写从 1000+ 次降到 chunk 数量级。
* 仅作用于"正常 poll DONE"路径;失败 batch / 单 batch 任务 / 其他
* 11 个 mergeCozeRowsIntoChunk 调用点保留原立即 merge 行为。
* 出现问题时可通过 AIIMAGE_SIMILAR_ASIN_COZE_RESULT_BUFFER_ENABLED=false
* 一键回滚到老路径。
*/
private boolean cozeResultBufferEnabled = true;
/**
* P0-4单 credential 抢 Coze 提交锁的最长等待时间(毫秒)。
* 原硬编码 1000ms在高并发 split retry 时大量抛 "Coze submit throttle lock timeout"
* 并把整批行 markFailed。应与 cozeSubmitMinIntervalMillis5000ms保持 1.5-2 倍关系,
* 默认 10000ms 给抢锁更多时间。
*/
private long cozeSubmitLockWaitMillis = 10000L;
/**
* P0-4抢 Coze 提交锁失败后下次重试间隔(毫秒)。
* 原硬编码 500ms会在指数退避算法中作为基础值500/1000/2000/4000ms 上限 4000
*/
private long cozeSubmitLockRetryDelayMillis = 500L;
@Data
public static class CozeCredential {
private String name;
private String workflowId;
private String token;
}
}

View File

@@ -3,6 +3,8 @@ package com.nanri.aiimage.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import java.nio.file.Path;
@Data
@ConfigurationProperties(prefix = "aiimage.storage")
public class StorageProperties {
@@ -12,4 +14,25 @@ public class StorageProperties {
private long sourceRetentionHours = 24;
private long resultRetentionHours = 24;
private long transientPayloadRetentionHours = 72;
public String getLocalTempDir() {
String configured = localTempDir == null ? "" : localTempDir.trim();
if (configured.isBlank()) {
configured = "./data/tmp";
}
try {
Path raw = Path.of(configured);
if (raw.isAbsolute()) {
return raw.normalize().toString();
}
} catch (Exception ignored) {
// Fall through to resolve against a stable base directory.
}
String userDir = System.getProperty("user.dir");
if (userDir == null || userDir.isBlank()) {
userDir = System.getProperty("java.io.tmpdir", ".");
}
return Path.of(userDir).resolve(configured).normalize().toString();
}
}

View File

@@ -9,6 +9,7 @@ import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Semaphore;
@Configuration
public class TaskFileJobConfig {
@@ -37,7 +38,23 @@ public class TaskFileJobConfig {
}
@Bean("cozeTaskExecutor")
public TaskExecutor cozeTaskExecutor(ExecutorService cozeVirtualThreadExecutor) {
return new ConcurrentTaskExecutor(cozeVirtualThreadExecutor);
public TaskExecutor cozeTaskExecutor(
ExecutorService cozeVirtualThreadExecutor,
@Value("${aiimage.coze-task.max-concurrent:12}") int maxConcurrent) {
Semaphore semaphore = new Semaphore(Math.max(1, maxConcurrent));
return new ConcurrentTaskExecutor(command -> cozeVirtualThreadExecutor.execute(() -> {
boolean acquired = false;
try {
semaphore.acquire();
acquired = true;
command.run();
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
} finally {
if (acquired) {
semaphore.release();
}
}
}));
}
}

View File

@@ -0,0 +1,17 @@
package com.nanri.aiimage.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
@Data
@ConfigurationProperties(prefix = "aiimage.task-image-cache-cleanup")
public class TaskImageCacheCleanupProperties {
private boolean enabled = true;
private String cron = "0 30 2 * * *";
private int retentionDays = 3;
private int batchSize = 5000;
private int maxBatchesPerRun = 200;
private long maxBytes = 2L * 1024L * 1024L * 1024L;
private long targetBytes = 1L * 1024L * 1024L * 1024L;
}

View File

@@ -20,6 +20,8 @@ import java.util.regex.Pattern;
@RequiredArgsConstructor
public class TaskOperationLockConfig implements WebMvcConfigurer {
private static final String LOCK_ATTRIBUTE = TaskOperationLockConfig.class.getName() + ".LOCK";
private final TaskDistributedLockService taskDistributedLockService;
@Override
@@ -28,15 +30,27 @@ public class TaskOperationLockConfig implements WebMvcConfigurer {
.addPathPatterns("/api/**");
}
public static boolean releaseRequestLock(HttpServletRequest request) {
if (request == null) {
return false;
}
Object lockHandle = request.getAttribute(LOCK_ATTRIBUTE);
if (lockHandle instanceof TaskDistributedLockService.LockHandle handle) {
request.removeAttribute(LOCK_ATTRIBUTE);
handle.close();
return true;
}
return false;
}
@Slf4j
private static final class TaskOperationLockInterceptor implements HandlerInterceptor {
private static final Pattern TASK_PATH = Pattern.compile(".*/api/([^/]+)/tasks/(\\d+)(?:/.*)?$");
private static final Pattern TASK_RESULT_PATH = Pattern.compile(".*/api/([^/]+)/tasks/(\\d+)/result/?$");
private static final Set<String> MUTATING_METHODS = Set.of("POST", "PUT", "PATCH", "DELETE");
private static final long RESULT_SUBMIT_WAIT_MILLIS = 10 * 60 * 1000L;
private static final long RESULT_SUBMIT_WAIT_MILLIS = 30 * 1000L;
private static final long DELETE_WAIT_MILLIS = 60 * 1000L;
private static final String LOCK_ATTRIBUTE = TaskOperationLockInterceptor.class.getName() + ".LOCK";
private final TaskDistributedLockService taskDistributedLockService;
@@ -50,6 +64,9 @@ public class TaskOperationLockConfig implements WebMvcConfigurer {
return true;
}
String uri = request.getRequestURI();
if ("POST".equals(request.getMethod()) && TASK_RESULT_PATH.matcher(uri == null ? "" : uri).matches()) {
return true;
}
Matcher matcher = TASK_PATH.matcher(uri == null ? "" : uri);
if (!matcher.matches()) {
return true;
@@ -83,10 +100,7 @@ public class TaskOperationLockConfig implements WebMvcConfigurer {
HttpServletResponse response,
Object handler,
Exception ex) {
Object lockHandle = request.getAttribute(LOCK_ATTRIBUTE);
if (lockHandle instanceof TaskDistributedLockService.LockHandle handle) {
handle.close();
}
releaseRequestLock(request);
}
}
}

View File

@@ -16,4 +16,26 @@ public class TransientStorageProperties {
private int readTimeoutSeconds = 60;
private int writeTimeoutSeconds = 60;
private int uploadMaxRetries = 3;
private int readMaxRetries = 3;
private int deleteMaxRetries = 3;
private int maxConcurrentUploads = 16;
private int maxConcurrentReads = 32;
private int maxConcurrentDeletes = 8;
private long acquirePermitTimeoutMillis = 2000;
private long baseRetryDelayMillis = 500;
private long maxRetryDelayMillis = 5000;
private long retryJitterMillis = 250;
private long failureWindowSeconds = 60;
private int failureWindowThreshold = 20;
private long failureCooldownMillis = 10000;
private int connectionPoolMaxIdle = 0;
private long connectionPoolKeepAliveMillis = 1;
private long warnPayloadBytes = 5L * 1024 * 1024;
private long maxPayloadBytes = 50L * 1024 * 1024;
private long maxStoredPayloadBytes = 50L * 1024 * 1024;
private boolean fallbackToLocalOnOversize = true;
private boolean deleteRetryEnabled = true;
private String deleteRetryCron = "0 */5 * * * *";
private int deleteRetryQueueCapacity = 10000;
private int deleteRetryBatchSize = 200;
}

View File

@@ -0,0 +1,76 @@
package com.nanri.aiimage.modules.admin.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.admin.model.dto.AdminUserCreateRequest;
import com.nanri.aiimage.modules.admin.model.dto.AdminUserUpdateRequest;
import com.nanri.aiimage.modules.admin.model.vo.AdminUserListVo;
import com.nanri.aiimage.modules.admin.service.AdminUserService;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.servlet.http.HttpServletRequest;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/admin")
@Tag(name = "管理后台-用户", description = "用户列表、创建、更新、删除")
public class AdminUserController {
private final AdminAuthSupport adminAuthSupport;
private final AdminUserService adminUserService;
@GetMapping("/users")
@Operation(summary = "分页查询用户列表")
public ApiResponse<AdminUserListVo> listUsers(HttpServletRequest request,
@RequestParam(required = false, defaultValue = "1") Integer page,
@RequestParam(name = "page_size", required = false, defaultValue = "15") Integer pageSize,
@RequestParam(required = false) String username,
@RequestParam(name = "search", required = false) String search,
@RequestParam(name = "created_by_id", required = false) Long createdById,
@RequestParam(name = "admin_id", required = false) Long adminId) {
AdminUserEntity currentUser = adminAuthSupport.requireAdmin(request);
String kw = (username == null || username.isBlank()) ? search : username;
Long filterCreatedBy = createdById != null ? createdById : adminId;
AdminUserListVo vo = adminUserService.listUsers(currentUser, page, pageSize, kw, filterCreatedBy);
return ApiResponse.success(vo);
}
@PostMapping("/user")
@Operation(summary = "创建用户")
public ApiResponse<Void> createUser(HttpServletRequest request,
@RequestBody AdminUserCreateRequest body) {
AdminUserEntity currentUser = adminAuthSupport.requireAdmin(request);
adminUserService.createUser(currentUser, body);
return ApiResponse.success("用户创建成功", null);
}
@PutMapping("/user/{uid}")
@Operation(summary = "更新用户")
public ApiResponse<Void> updateUser(HttpServletRequest request,
@PathVariable Long uid,
@RequestBody AdminUserUpdateRequest body) {
AdminUserEntity currentUser = adminAuthSupport.requireAdmin(request);
adminUserService.updateUser(currentUser, uid, body);
return ApiResponse.success("更新成功", null);
}
@DeleteMapping("/user/{uid}")
@Operation(summary = "删除用户")
public ApiResponse<Void> deleteUser(HttpServletRequest request,
@PathVariable Long uid) {
AdminUserEntity currentUser = adminAuthSupport.requireAdmin(request);
adminUserService.deleteUser(currentUser, uid);
return ApiResponse.success("删除成功", null);
}
}

Some files were not shown because too many files have changed in this diff Show More