Compare commits
53 Commits
fae26aa460
..
master
| Author | SHA1 | Date | |
|---|---|---|---|
| 5b8105ec2b | |||
| 5fcb449946 | |||
| ff68426c69 | |||
| 46be044121 | |||
| e0303f9cba | |||
| 9d39705c77 | |||
| 986df86e89 | |||
| bd359411a9 | |||
| 3634ea1d62 | |||
| 3ce0569c59 | |||
| d2d95f0b71 | |||
| 2a51006888 | |||
| a89de129ea | |||
| ddefcbed56 | |||
| 3137299bfe | |||
| 1fe3368c5a | |||
| 7aea3a0a50 | |||
| 188aedec84 | |||
| d5952945dd | |||
| 6e1689dfe5 | |||
| 4f16a02658 | |||
| 5e9a59b327 | |||
| ab09cff427 | |||
| 47a9520a82 | |||
| 7a6c3c3fa3 | |||
| 803b5d583d | |||
| 1c52cd529b | |||
| ab28b168ec | |||
| 367b4b7553 | |||
| 25c7323c47 | |||
| aea0e16279 | |||
| f566573fce | |||
| 8fcceb3226 | |||
| ed0d6575c8 | |||
| f2ada02383 | |||
| 5e5816cd74 | |||
| d189d94c3b | |||
| 79b5d40327 | |||
| 228d481211 | |||
| d6f8368493 | |||
| b0f764b6b6 | |||
| 1403fec5fe | |||
| 07b4ebe983 | |||
| 0b2b9303d0 | |||
| 05a2c479a5 | |||
| 52b55df7b2 | |||
| b54f72d3f6 | |||
| 24c5a09c7f | |||
| 8cab9d4bad | |||
| 5ea52e5291 | |||
| 9166656673 | |||
| 1360a44e01 | |||
| 86c05e71a2 |
@@ -0,0 +1,44 @@
|
|||||||
|
import { expect, test, type Page } from '@playwright/test'
|
||||||
|
|
||||||
|
// 后台管理列表「创建时间 + 更新时间」两列验收(2026-09-19 需求):
|
||||||
|
// 8 个实体管理列表统一展示创建/更新时间,数据取自各列表 VO。
|
||||||
|
// mock 夹具统一给 createdAt=2026-03-01 09:15:00、updatedAt=2026-09-18 16:42:30,
|
||||||
|
// 页面统一经 formatDateTime 归一为「YYYY-MM-DD HH:mm:ss」。
|
||||||
|
|
||||||
|
const CREATED = '2026-03-01 09:15:00'
|
||||||
|
const UPDATED = '2026-09-18 16:42:30'
|
||||||
|
|
||||||
|
const PAGES: Array<{ title: string; path: string; heading: string }> = [
|
||||||
|
{ title: '用户管理', path: '/admin-vue/account/users', heading: '用户列表' },
|
||||||
|
{ title: '菜单管理', path: '/admin-vue/account/menus', heading: '菜单' },
|
||||||
|
{ title: '密钥管理', path: '/admin-vue/account/user-secrets', heading: '用户密钥列表' },
|
||||||
|
{ title: '不符合ASIN数据', path: '/admin-vue/asin-center/invalid', heading: '品牌数据列表' },
|
||||||
|
{ title: '数据去重总数据', path: '/admin-vue/asin-center/registry', heading: '数据去重总数据' },
|
||||||
|
{ title: '查询ASIN', path: '/admin-vue/asin-center/query', heading: '查询' },
|
||||||
|
{ title: '最低价ASIN', path: '/admin-vue/asin-center/skip-price', heading: '最低价' },
|
||||||
|
{ title: '商品类目', path: '/admin-vue/asin-center/categories', heading: '商品类目' },
|
||||||
|
]
|
||||||
|
|
||||||
|
async function openPage(page: Page, path: string) {
|
||||||
|
await page.goto(path)
|
||||||
|
await expect(page.locator('.panel-box table tbody tr').first()).toBeVisible({ timeout: 15000 })
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const spec of PAGES) {
|
||||||
|
test(`${spec.title}:表格展示创建时间与更新时间`, async ({ page }) => {
|
||||||
|
const errors: string[] = []
|
||||||
|
page.on('pageerror', (error) => errors.push(error.message))
|
||||||
|
|
||||||
|
await openPage(page, spec.path)
|
||||||
|
|
||||||
|
const table = page.locator('.panel-box table').first()
|
||||||
|
// 表头出现两列
|
||||||
|
await expect(table.locator('thead')).toContainText('创建时间')
|
||||||
|
await expect(table.locator('thead')).toContainText('更新时间')
|
||||||
|
// 数据行渲染出真实时间值(非空、非占位符)
|
||||||
|
await expect(table.locator('tbody')).toContainText(CREATED)
|
||||||
|
await expect(table.locator('tbody')).toContainText(UPDATED)
|
||||||
|
|
||||||
|
expect(errors).toEqual([])
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
import { expect, test, type Page } from '@playwright/test'
|
||||||
|
|
||||||
|
// 教程管理页验收(module 记录与版本):列表新增「版本号」列 + 上传时间列,
|
||||||
|
// 默认按上传时间降序;版本号/上传时间表头可点击切换升降序;空版本历史行沉底。
|
||||||
|
// 依赖 scripts/mock-admin-server.mjs 的教程包夹具(4 条,其中 1 条无版本号、1 条 09-01 历史行)。
|
||||||
|
|
||||||
|
async function openTutorial(page: Page) {
|
||||||
|
await page.goto('/admin-vue/records/tutorial')
|
||||||
|
await expect(page.locator('.admin-topbar h1')).toHaveText('教程管理')
|
||||||
|
await expect(page.locator('.panel-box tbody tr').first()).toBeVisible()
|
||||||
|
}
|
||||||
|
|
||||||
|
const versionTexts = (page: Page) =>
|
||||||
|
page.locator('.panel-box tbody tr td:nth-child(3)').allTextContents()
|
||||||
|
|
||||||
|
const fileTexts = (page: Page) =>
|
||||||
|
page.locator('.panel-box tbody tr td:nth-child(2) .file-name').allTextContents()
|
||||||
|
|
||||||
|
test('test_tutorial_list_default_desc_by_upload_time', async ({ page }) => {
|
||||||
|
await openTutorial(page)
|
||||||
|
|
||||||
|
// 表头:版本号、上传时间(均带排序标记)
|
||||||
|
await expect(page.locator('.sort-version')).toHaveText(/版本号/)
|
||||||
|
await expect(page.locator('.sort-time')).toHaveText(/上传时间/)
|
||||||
|
// 默认排序状态:上传时间降序(▼),版本号未激活(▲▼)
|
||||||
|
await expect(page.locator('.sort-time .sort-mark')).toHaveText('▼')
|
||||||
|
await expect(page.locator('.sort-version .sort-mark')).toHaveText('▲▼')
|
||||||
|
|
||||||
|
// 默认按上传时间降序:09-14 → 09-12 → 09-10 → 09-01;空版本行沉底
|
||||||
|
expect(await versionTexts(page)).toEqual(['v3.10.0', 'v3.2.0', 'v3.1.0', '—'])
|
||||||
|
expect(await fileTexts(page)).toEqual([
|
||||||
|
'数富AI-教学客户端-v4.zip',
|
||||||
|
'数富AI-教学客户端-v3.zip',
|
||||||
|
'数富AI-教学客户端-v2.zip',
|
||||||
|
'数富AI-教学客户端.zip',
|
||||||
|
])
|
||||||
|
// 当前生效 = 最新上传(夹具里 09-14 那条),不随列表排序变化
|
||||||
|
await expect(page.locator('.tag-active')).toHaveCount(1)
|
||||||
|
await expect(page.locator('.tag-active').locator('xpath=..')).toHaveText(/数富AI-教学客户端-v4\.zip/)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('test_tutorial_sort_toggle_by_version_and_time', async ({ page }) => {
|
||||||
|
await openTutorial(page)
|
||||||
|
|
||||||
|
// 点「版本号」:首次为降序(v3.10.0 按自然序大于 v3.2.0),空版本行仍在末尾
|
||||||
|
await page.locator('.sort-version').click()
|
||||||
|
await expect(page.locator('.sort-version .sort-mark')).toHaveText('▼')
|
||||||
|
expect(await versionTexts(page)).toEqual(['v3.10.0', 'v3.2.0', 'v3.1.0', '—'])
|
||||||
|
|
||||||
|
// 再点一次切升序
|
||||||
|
await page.locator('.sort-version').click()
|
||||||
|
await expect(page.locator('.sort-version .sort-mark')).toHaveText('▲')
|
||||||
|
expect(await versionTexts(page)).toEqual(['v3.1.0', 'v3.2.0', 'v3.10.0', '—'])
|
||||||
|
|
||||||
|
// 点回「上传时间」:切换字段时回到降序默认
|
||||||
|
await page.locator('.sort-time').click()
|
||||||
|
await expect(page.locator('.sort-time .sort-mark')).toHaveText('▼')
|
||||||
|
await expect(page.locator('.sort-version .sort-mark')).toHaveText('▲▼')
|
||||||
|
expect(await fileTexts(page)).toEqual([
|
||||||
|
'数富AI-教学客户端-v4.zip',
|
||||||
|
'数富AI-教学客户端-v3.zip',
|
||||||
|
'数富AI-教学客户端-v2.zip',
|
||||||
|
'数富AI-教学客户端.zip',
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
test('test_tutorial_upload_dialog_has_version_field', async ({ page }) => {
|
||||||
|
await openTutorial(page)
|
||||||
|
|
||||||
|
await page.getByRole('button', { name: '上传教程包' }).click()
|
||||||
|
const dialog = page.locator('.el-dialog')
|
||||||
|
await expect(dialog).toBeVisible()
|
||||||
|
await expect(dialog.locator('.el-form-item').first()).toContainText('版本号')
|
||||||
|
await expect(dialog.locator('input').first()).toHaveAttribute('maxlength', '64')
|
||||||
|
|
||||||
|
// 不选文件直接提交:提示仍以 zip 为必填(版本号可空)
|
||||||
|
await dialog.getByRole('button', { name: '上传教程包' }).click()
|
||||||
|
await expect(dialog.locator('.el-alert')).toContainText('请选择 zip 压缩包')
|
||||||
|
})
|
||||||
@@ -142,6 +142,15 @@ function json(res, payload, status = 200) {
|
|||||||
res.end(body)
|
res.end(body)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ===================== 教程包夹具(教程管理页:版本号列 + 上传时间排序验收用) ===================== */
|
||||||
|
// 故意乱序给出,且含一条无版本号的历史行:页面默认应按上传时间降序、空版本行沉底。
|
||||||
|
const TUTORIAL_PACKAGES = [
|
||||||
|
{ id: 3, file_name: '数富AI-教学客户端-v3.zip', version: 'v3.2.0', object_key: 'tutorial/20260912090000-数富AI-教学客户端-v3.zip', file_size: 1048576, file_url: 'https://oss.aishufu.top/client/tutorial/t3.zip', created_at: '2026-09-12 09:00' },
|
||||||
|
{ id: 1, file_name: '数富AI-教学客户端.zip', version: '', object_key: 'tutorial/数富AI-教学客户端.zip', file_size: 0, file_url: 'https://oss.aishufu.top/client/tutorial/legacy.zip', created_at: '2026-09-01 08:00' },
|
||||||
|
{ id: 4, file_name: '数富AI-教学客户端-v4.zip', version: 'v3.10.0', object_key: 'tutorial/20260914103000-数富AI-教学客户端-v4.zip', file_size: 2097152, file_url: 'https://oss.aishufu.top/client/tutorial/t4.zip', created_at: '2026-09-14 10:30' },
|
||||||
|
{ id: 2, file_name: '数富AI-教学客户端-v2.zip', version: 'v3.1.0', object_key: 'tutorial/20260910120000-数富AI-教学客户端-v2.zip', file_size: 524288, file_url: 'https://oss.aishufu.top/client/tutorial/t2.zip', created_at: '2026-09-10 12:00' },
|
||||||
|
]
|
||||||
|
|
||||||
/* ===================== 站内通知夹具(铃铛面板:搜索/按天分组/分页验收用) ===================== */
|
/* ===================== 站内通知夹具(铃铛面板:搜索/按天分组/分页验收用) ===================== */
|
||||||
const NOTIFICATIONS = [
|
const NOTIFICATIONS = [
|
||||||
...[6, 5, 4, 3, 2, 1].map((seq) => notif(seq, '2026-09-13', 22, seq, seq <= 3)),
|
...[6, 5, 4, 3, 2, 1].map((seq) => notif(seq, '2026-09-13', 22, seq, seq <= 3)),
|
||||||
@@ -211,13 +220,183 @@ const server = createServer((req, res) => {
|
|||||||
return json(res, {
|
return json(res, {
|
||||||
success: true,
|
success: true,
|
||||||
data: {
|
data: {
|
||||||
items: [{ id: 1, username: 'admin', role: 'super_admin', creatorUsername: 'system', createdAt: '2026-01-01 00:00:00' }],
|
items: [{ id: 1, username: 'admin', role: 'super_admin', creatorUsername: 'system', created_at: '2026-03-01 09:15:00', updated_at: '2026-09-18 16:42:30' }],
|
||||||
total: 1,
|
total: 1,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if (url === '/api/admin/permission-menus') {
|
if (url === '/api/admin/permission-menus') {
|
||||||
return json(res, { success: true, data: { items: [] } })
|
// 菜单管理列表:一条根节点,带创建/更新时间(验收时间列)。
|
||||||
|
return json(res, {
|
||||||
|
success: true,
|
||||||
|
data: [
|
||||||
|
{
|
||||||
|
id: 1,
|
||||||
|
name: '用户管理',
|
||||||
|
column_key: 'admin_users',
|
||||||
|
parent_id: null,
|
||||||
|
menu_type: 'admin',
|
||||||
|
route_path: 'account/users',
|
||||||
|
sort_order: 10,
|
||||||
|
created_at: '2026-03-01T09:15:00',
|
||||||
|
updated_at: '2026-09-18T16:42:30',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if (url === '/api/admin/invalid-asin-data') {
|
||||||
|
return json(res, {
|
||||||
|
success: true,
|
||||||
|
data: {
|
||||||
|
items: [
|
||||||
|
{
|
||||||
|
id: 1,
|
||||||
|
dataValue: 'B0INVALID1',
|
||||||
|
brand: '测试品牌',
|
||||||
|
groupId: 1,
|
||||||
|
groupName: '测试分组',
|
||||||
|
recordSource: 'MANUAL',
|
||||||
|
createdAt: '2026-03-01 09:15:00',
|
||||||
|
updatedAt: '2026-09-18 16:42:30',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
total: 1,
|
||||||
|
page: 1,
|
||||||
|
pageSize: 10,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if (url === '/api/admin/dedupe-total-data') {
|
||||||
|
return json(res, {
|
||||||
|
success: true,
|
||||||
|
data: {
|
||||||
|
items: [
|
||||||
|
{
|
||||||
|
id: 1,
|
||||||
|
dataValue: 'B0DEDUPE01',
|
||||||
|
country: 'DE',
|
||||||
|
groupId: 1,
|
||||||
|
groupName: '测试分组',
|
||||||
|
uploaderUserId: 1,
|
||||||
|
username: 'admin',
|
||||||
|
createdAt: '2026-03-01 09:15:00',
|
||||||
|
updatedAt: '2026-09-18 16:42:30',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
total: 1,
|
||||||
|
page: 1,
|
||||||
|
pageSize: 10,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if (url === '/api/admin/query-asins') {
|
||||||
|
return json(res, {
|
||||||
|
success: true,
|
||||||
|
data: {
|
||||||
|
items: [
|
||||||
|
{
|
||||||
|
id: 1,
|
||||||
|
groupId: 1,
|
||||||
|
groupName: '测试分组',
|
||||||
|
shopName: '测试店铺',
|
||||||
|
asinDe: 'B0QUERY001',
|
||||||
|
asinUk: '',
|
||||||
|
asinFr: '',
|
||||||
|
asinIt: '',
|
||||||
|
asinEs: '',
|
||||||
|
createdAt: '2026-03-01 09:15:00',
|
||||||
|
updatedAt: '2026-09-18 16:42:30',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
total: 1,
|
||||||
|
page: 1,
|
||||||
|
pageSize: 10,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if (url === '/api/admin/skip-price-asins') {
|
||||||
|
return json(res, {
|
||||||
|
success: true,
|
||||||
|
data: {
|
||||||
|
items: [
|
||||||
|
{
|
||||||
|
id: 1,
|
||||||
|
groupId: 1,
|
||||||
|
groupName: '测试分组',
|
||||||
|
shopName: '测试店铺',
|
||||||
|
asinDe: 'B0SKIP0001',
|
||||||
|
minimumPriceDe: 9.99,
|
||||||
|
asinUk: '',
|
||||||
|
minimumPriceUk: null,
|
||||||
|
asinFr: '',
|
||||||
|
minimumPriceFr: null,
|
||||||
|
asinIt: '',
|
||||||
|
minimumPriceIt: null,
|
||||||
|
asinEs: '',
|
||||||
|
minimumPriceEs: null,
|
||||||
|
createdAt: '2026-03-01 09:15:00',
|
||||||
|
updatedAt: '2026-09-18 16:42:30',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
total: 1,
|
||||||
|
page: 1,
|
||||||
|
pageSize: 10,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if (url === '/api/admin/product-categories/children') {
|
||||||
|
return json(res, {
|
||||||
|
success: true,
|
||||||
|
data: {
|
||||||
|
items: [
|
||||||
|
{
|
||||||
|
id: 1,
|
||||||
|
parentId: null,
|
||||||
|
name: '护肤品',
|
||||||
|
categoryKey: 'skincare',
|
||||||
|
sortOrder: 10,
|
||||||
|
description: '测试备注',
|
||||||
|
isBuiltin: true,
|
||||||
|
childCount: 0,
|
||||||
|
level: 0,
|
||||||
|
path: '护肤品',
|
||||||
|
createdAt: '2026-03-01 09:15:00',
|
||||||
|
updatedAt: '2026-09-18 16:42:30',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
total: 1,
|
||||||
|
page: 1,
|
||||||
|
pageSize: 10,
|
||||||
|
hasMore: false,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if (url === '/api/admin/user-secrets') {
|
||||||
|
const emptyModule = { moduleKey: '', moduleLabel: '', masked: '', full: '', exists: false, checkStatus: 'unknown', checkCode: '', checkMessage: '', checkLatencyMs: null, checkedAt: null, updatedAt: null }
|
||||||
|
return json(res, {
|
||||||
|
success: true,
|
||||||
|
data: {
|
||||||
|
items: [
|
||||||
|
{
|
||||||
|
userId: 1,
|
||||||
|
username: 'admin',
|
||||||
|
groups: [],
|
||||||
|
leaderUsername: '',
|
||||||
|
similarAsin: { ...emptyModule, moduleKey: 'similar-asin', moduleLabel: '货源查询密钥' },
|
||||||
|
appearancePatent: { ...emptyModule, moduleKey: 'appearance-patent', moduleLabel: '外观专利密钥' },
|
||||||
|
proxy: { ...emptyModule, moduleKey: 'proxy', moduleLabel: '代理设置' },
|
||||||
|
status: 'unknown',
|
||||||
|
statusMessage: '',
|
||||||
|
createdAt: '2026-03-01 09:15:00',
|
||||||
|
updatedAt: '2026-09-18 16:42:30',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
total: 1,
|
||||||
|
page: 1,
|
||||||
|
pageSize: 15,
|
||||||
|
groupOptions: [],
|
||||||
|
},
|
||||||
|
})
|
||||||
}
|
}
|
||||||
if (url === '/api/admin/shop-manage-groups') {
|
if (url === '/api/admin/shop-manage-groups') {
|
||||||
return json(res, { success: true, data: { items: [] } })
|
return json(res, { success: true, data: { items: [] } })
|
||||||
@@ -243,6 +422,11 @@ const server = createServer((req, res) => {
|
|||||||
if (url === '/api/admin/notifications') {
|
if (url === '/api/admin/notifications') {
|
||||||
return json(res, notificationPage(new URL(req.url || '/', 'http://127.0.0.1').searchParams))
|
return json(res, notificationPage(new URL(req.url || '/', 'http://127.0.0.1').searchParams))
|
||||||
}
|
}
|
||||||
|
if (url === '/api/admin/tutorials') {
|
||||||
|
// 与 Java 侧同契约:created_at 降序返回(页面"当前生效"取第一条)。
|
||||||
|
const items = [...TUTORIAL_PACKAGES].sort((a, b) => (a.created_at < b.created_at ? 1 : -1))
|
||||||
|
return json(res, { success: true, data: { items } })
|
||||||
|
}
|
||||||
if (url.startsWith('/api/')) {
|
if (url.startsWith('/api/')) {
|
||||||
return json(res, { success: true, data: { items: [], total: 0 } })
|
return json(res, { success: true, data: { items: [], total: 0 } })
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,135 @@
|
|||||||
|
import { http } from './http'
|
||||||
|
import { unwrap } from './envelope'
|
||||||
|
|
||||||
|
/** 日志文件行(桌面客户端 / 麦象采集机上报)。 */
|
||||||
|
export interface DeviceLogFileRow {
|
||||||
|
id: number
|
||||||
|
source: string
|
||||||
|
deviceId: string
|
||||||
|
deviceName: string | null
|
||||||
|
username: string | null
|
||||||
|
uid: number | null
|
||||||
|
fileName: string
|
||||||
|
logDate: string
|
||||||
|
uploadedBytes: number
|
||||||
|
partCount: number
|
||||||
|
lastUploadAt: string | null
|
||||||
|
createdAt: string | null
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DeviceLogPage {
|
||||||
|
items: DeviceLogFileRow[]
|
||||||
|
total: number
|
||||||
|
page: number
|
||||||
|
pageSize: number
|
||||||
|
/** 云端日志保留天数(页面提示用)。 */
|
||||||
|
retentionDays: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DeviceLogContent {
|
||||||
|
fileId: number
|
||||||
|
fileName: string
|
||||||
|
content: string
|
||||||
|
totalBytes: number
|
||||||
|
shownBytes: number
|
||||||
|
truncated: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DeviceLogDevice {
|
||||||
|
source: string
|
||||||
|
deviceId: string
|
||||||
|
deviceName: string | null
|
||||||
|
lastUploadAt: string | null
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DeviceLogOverride {
|
||||||
|
id: number
|
||||||
|
source: string
|
||||||
|
deviceId: string
|
||||||
|
deviceName: string | null
|
||||||
|
mode: string
|
||||||
|
updatedAt: string | null
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DeviceLogConfigData {
|
||||||
|
globalMode: string
|
||||||
|
overrides: DeviceLogOverride[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DeviceLogQuery {
|
||||||
|
source?: string
|
||||||
|
keyword?: string
|
||||||
|
startDate?: string
|
||||||
|
endDate?: string
|
||||||
|
page: number
|
||||||
|
pageSize: number
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 分页查询日志文件列表:GET /api/admin/device-logs/files */
|
||||||
|
export async function fetchDeviceLogFiles(params: DeviceLogQuery): Promise<DeviceLogPage> {
|
||||||
|
const query: Record<string, string | number> = { page: params.page, pageSize: params.pageSize }
|
||||||
|
if (params.source) query.source = params.source
|
||||||
|
if (params.keyword) query.keyword = params.keyword
|
||||||
|
if (params.startDate) query.startDate = params.startDate
|
||||||
|
if (params.endDate) query.endDate = params.endDate
|
||||||
|
const { data } = await http.get('/api/admin/device-logs/files', { params: query })
|
||||||
|
return unwrap<DeviceLogPage>(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 查看日志尾部内容:GET /api/admin/device-logs/content */
|
||||||
|
export async function fetchDeviceLogContent(fileId: number, maxBytes?: number): Promise<DeviceLogContent> {
|
||||||
|
const query: Record<string, number> = { fileId }
|
||||||
|
if (maxBytes) query.maxBytes = maxBytes
|
||||||
|
const { data } = await http.get('/api/admin/device-logs/content', { params: query })
|
||||||
|
return unwrap<DeviceLogContent>(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 完整日志下载地址(同域 cookie 鉴权,直接给 a[href] 或 window.open 用)。 */
|
||||||
|
export function deviceLogDownloadUrl(fileId: number): string {
|
||||||
|
return `/api/admin/device-logs/download?fileId=${fileId}`
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 删除日志文件(片段与元数据,不可恢复):DELETE /api/admin/device-logs/{id} */
|
||||||
|
export async function deleteDeviceLogFile(id: number): Promise<void> {
|
||||||
|
const { data } = await http.delete(`/api/admin/device-logs/${id}`)
|
||||||
|
unwrap<unknown>(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 采集配置(全局默认 + 终端覆盖):GET /api/admin/device-logs/config */
|
||||||
|
export async function fetchDeviceLogConfig(keyword?: string): Promise<DeviceLogConfigData> {
|
||||||
|
const { data } = await http.get('/api/admin/device-logs/config', {
|
||||||
|
params: keyword ? { keyword } : undefined,
|
||||||
|
})
|
||||||
|
return unwrap<DeviceLogConfigData>(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 最近上报过的终端(覆盖选择用):GET /api/admin/device-logs/devices */
|
||||||
|
export async function fetchDeviceLogDevices(): Promise<DeviceLogDevice[]> {
|
||||||
|
const { data } = await http.get('/api/admin/device-logs/devices')
|
||||||
|
return unwrap<DeviceLogDevice[]>(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 设置全局采集模式:PUT /api/admin/device-logs/config/global */
|
||||||
|
export async function updateDeviceLogGlobalMode(mode: string): Promise<void> {
|
||||||
|
const { data } = await http.put('/api/admin/device-logs/config/global', undefined, { params: { mode } })
|
||||||
|
unwrap<unknown>(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 设置/更新终端覆盖:PUT /api/admin/device-logs/config/device */
|
||||||
|
export async function updateDeviceLogOverride(
|
||||||
|
source: string,
|
||||||
|
deviceId: string,
|
||||||
|
deviceName: string | null,
|
||||||
|
mode: string,
|
||||||
|
): Promise<void> {
|
||||||
|
const { data } = await http.put('/api/admin/device-logs/config/device', undefined, {
|
||||||
|
params: { source, deviceId, deviceName: deviceName || undefined, mode },
|
||||||
|
})
|
||||||
|
unwrap<unknown>(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 删除终端覆盖(回落到全局默认):DELETE /api/admin/device-logs/config/device/{id} */
|
||||||
|
export async function deleteDeviceLogOverride(id: number): Promise<void> {
|
||||||
|
const { data } = await http.delete(`/api/admin/device-logs/config/device/${id}`)
|
||||||
|
unwrap<unknown>(data)
|
||||||
|
}
|
||||||
@@ -30,6 +30,8 @@ export interface AdminUserSecretRow {
|
|||||||
proxy: AdminUserSecretModule
|
proxy: AdminUserSecretModule
|
||||||
status: string
|
status: string
|
||||||
statusMessage: string
|
statusMessage: string
|
||||||
|
/** 首次配置时间(三模块中最早);从未配置为 null。 */
|
||||||
|
createdAt: string | null
|
||||||
updatedAt: string | null
|
updatedAt: string | null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -23,6 +23,8 @@ export function toAdminUserItem(raw: unknown): AdminUser | null {
|
|||||||
if (createdById !== null) item.createdById = createdById
|
if (createdById !== null) item.createdById = createdById
|
||||||
const createdAt = text(r.created_at)
|
const createdAt = text(r.created_at)
|
||||||
if (createdAt) item.createdAt = createdAt
|
if (createdAt) item.createdAt = createdAt
|
||||||
|
const updatedAt = text(r.updated_at)
|
||||||
|
if (updatedAt) item.updatedAt = updatedAt
|
||||||
const creator = text(r.creator_username)
|
const creator = text(r.creator_username)
|
||||||
if (creator) item.creatorUsername = creator
|
if (creator) item.creatorUsername = creator
|
||||||
const abbr = text(r.pinyin_abbr)
|
const abbr = text(r.pinyin_abbr)
|
||||||
|
|||||||
@@ -241,6 +241,7 @@ onMounted(loadMenus)
|
|||||||
<th style="width: 100px">菜单类型</th>
|
<th style="width: 100px">菜单类型</th>
|
||||||
<th style="width: 120px">上级菜单</th>
|
<th style="width: 120px">上级菜单</th>
|
||||||
<th style="width: 170px">创建时间</th>
|
<th style="width: 170px">创建时间</th>
|
||||||
|
<th style="width: 170px">更新时间</th>
|
||||||
<th style="width: 170px">操作</th>
|
<th style="width: 170px">操作</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
@@ -271,6 +272,7 @@ onMounted(loadMenus)
|
|||||||
<td><span class="menu-type-text">{{ menuTypeLabel(row.menuType) }}</span></td>
|
<td><span class="menu-type-text">{{ menuTypeLabel(row.menuType) }}</span></td>
|
||||||
<td>{{ parentNameOf(row) }}</td>
|
<td>{{ parentNameOf(row) }}</td>
|
||||||
<td>{{ formatDateTime(row.createdAt) }}</td>
|
<td>{{ formatDateTime(row.createdAt) }}</td>
|
||||||
|
<td>{{ formatDateTime(row.updatedAt) }}</td>
|
||||||
<td class="ops-cell">
|
<td class="ops-cell">
|
||||||
<button class="btn btn-sm" type="button" @click="openEdit(row)">编辑</button>
|
<button class="btn btn-sm" type="button" @click="openEdit(row)">编辑</button>
|
||||||
<button
|
<button
|
||||||
@@ -286,10 +288,10 @@ onMounted(loadMenus)
|
|||||||
</tr>
|
</tr>
|
||||||
</template>
|
</template>
|
||||||
<tr v-else-if="loading">
|
<tr v-else-if="loading">
|
||||||
<td colspan="7" class="empty-tip">加载中...</td>
|
<td colspan="8" class="empty-tip">加载中...</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr v-else>
|
<tr v-else>
|
||||||
<td colspan="7" class="empty-tip">{{ filterName || filterType ? '暂无匹配菜单' : '暂无菜单,请先在上方新增' }}</td>
|
<td colspan="8" class="empty-tip">{{ filterName || filterType ? '暂无匹配菜单' : '暂无菜单,请先在上方新增' }}</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
@@ -77,7 +77,7 @@ watch(
|
|||||||
node-key="id"
|
node-key="id"
|
||||||
show-checkbox
|
show-checkbox
|
||||||
default-expand-all
|
default-expand-all
|
||||||
:props="{ label: 'name', children: 'children' }"
|
:props="{ label: 'name', children: 'children', disabled: 'disabled' }"
|
||||||
@check="onCheck"
|
@check="onCheck"
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
@@ -89,7 +89,7 @@ watch(
|
|||||||
node-key="id"
|
node-key="id"
|
||||||
show-checkbox
|
show-checkbox
|
||||||
default-expand-all
|
default-expand-all
|
||||||
:props="{ label: 'name', children: 'children' }"
|
:props="{ label: 'name', children: 'children', disabled: 'disabled' }"
|
||||||
@check="onCheck"
|
@check="onCheck"
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -233,6 +233,8 @@ onMounted(load)
|
|||||||
<th style="width: 230px">外观专利密钥</th>
|
<th style="width: 230px">外观专利密钥</th>
|
||||||
<th style="width: 380px">代理设置</th>
|
<th style="width: 380px">代理设置</th>
|
||||||
<th style="width: 130px">状态</th>
|
<th style="width: 130px">状态</th>
|
||||||
|
<th style="width: 170px">创建时间</th>
|
||||||
|
<th style="width: 170px">更新时间</th>
|
||||||
<th style="width: 170px">操作</th>
|
<th style="width: 170px">操作</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
@@ -278,6 +280,8 @@ onMounted(load)
|
|||||||
{{ rowStatusMeta(row.status).label }}
|
{{ rowStatusMeta(row.status).label }}
|
||||||
</span>
|
</span>
|
||||||
</td>
|
</td>
|
||||||
|
<td>{{ formatDateTime(row.createdAt) }}</td>
|
||||||
|
<td>{{ formatDateTime(row.updatedAt) }}</td>
|
||||||
<td class="ops-cell">
|
<td class="ops-cell">
|
||||||
<button
|
<button
|
||||||
class="btn btn-sm"
|
class="btn btn-sm"
|
||||||
@@ -292,10 +296,10 @@ onMounted(load)
|
|||||||
</tr>
|
</tr>
|
||||||
</template>
|
</template>
|
||||||
<tr v-else-if="loading">
|
<tr v-else-if="loading">
|
||||||
<td :colspan="isSuperAdmin ? 8 : 7" class="empty-tip">加载中...</td>
|
<td :colspan="isSuperAdmin ? 10 : 9" class="empty-tip">加载中...</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr v-else>
|
<tr v-else>
|
||||||
<td :colspan="isSuperAdmin ? 8 : 7" class="empty-tip">{{ keyword || statusFilter || groupFilter ? '暂无匹配记录' : '暂无用户密钥记录' }}</td>
|
<td :colspan="isSuperAdmin ? 10 : 9" class="empty-tip">{{ keyword || statusFilter || groupFilter ? '暂无匹配记录' : '暂无用户密钥记录' }}</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
@@ -169,6 +169,7 @@ onMounted(loadUsers)
|
|||||||
<th style="width: 140px">角色</th>
|
<th style="width: 140px">角色</th>
|
||||||
<th style="width: 160px">所属管理员</th>
|
<th style="width: 160px">所属管理员</th>
|
||||||
<th style="width: 180px">创建时间</th>
|
<th style="width: 180px">创建时间</th>
|
||||||
|
<th style="width: 180px">更新时间</th>
|
||||||
<th style="width: 160px">操作</th>
|
<th style="width: 160px">操作</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
@@ -180,6 +181,7 @@ onMounted(loadUsers)
|
|||||||
<td>{{ roleLabel(row.role) }}</td>
|
<td>{{ roleLabel(row.role) }}</td>
|
||||||
<td>{{ row.creatorUsername || '-' }}</td>
|
<td>{{ row.creatorUsername || '-' }}</td>
|
||||||
<td>{{ formatDateTime(row.createdAt) }}</td>
|
<td>{{ formatDateTime(row.createdAt) }}</td>
|
||||||
|
<td>{{ formatDateTime(row.updatedAt) }}</td>
|
||||||
<td class="ops-cell">
|
<td class="ops-cell">
|
||||||
<button
|
<button
|
||||||
class="btn btn-sm"
|
class="btn btn-sm"
|
||||||
@@ -203,10 +205,10 @@ onMounted(loadUsers)
|
|||||||
</tr>
|
</tr>
|
||||||
</template>
|
</template>
|
||||||
<tr v-else-if="loading">
|
<tr v-else-if="loading">
|
||||||
<td colspan="6" class="empty-tip">加载中...</td>
|
<td colspan="7" class="empty-tip">加载中...</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr v-else>
|
<tr v-else>
|
||||||
<td colspan="6" class="empty-tip">{{ userListEmptyHint() }}</td>
|
<td colspan="7" class="empty-tip">{{ userListEmptyHint() }}</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ export interface MenuManageNode {
|
|||||||
routePath: string
|
routePath: string
|
||||||
sortOrder: number
|
sortOrder: number
|
||||||
createdAt?: string
|
createdAt?: string
|
||||||
|
updatedAt?: string
|
||||||
children?: MenuManageNode[]
|
children?: MenuManageNode[]
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -90,6 +91,8 @@ export function parseMenuManageItem(raw: unknown): MenuManageNode | null {
|
|||||||
if (rootColumnKey) node.rootColumnKey = rootColumnKey
|
if (rootColumnKey) node.rootColumnKey = rootColumnKey
|
||||||
const createdAt = text(record.created_at)
|
const createdAt = text(record.created_at)
|
||||||
if (createdAt) node.createdAt = createdAt
|
if (createdAt) node.createdAt = createdAt
|
||||||
|
const updatedAt = text(record.updated_at)
|
||||||
|
if (updatedAt) node.updatedAt = updatedAt
|
||||||
return node
|
return node
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,6 +16,11 @@ export interface MenuOptionNode {
|
|||||||
parentId: number | null
|
parentId: number | null
|
||||||
/** 所属菜单类型(admin 后台 / app 前端客户端),提交时按类型分区落库。 */
|
/** 所属菜单类型(admin 后台 / app 前端客户端),提交时按类型分区落库。 */
|
||||||
type: string
|
type: string
|
||||||
|
/**
|
||||||
|
* 当前操作者无权授予(后端 grantable=false)时置灰:仍展示并回显已勾选,
|
||||||
|
* 但不允许改勾选。非超管只能授自己已有的菜单,勾到越权项会让整笔保存回滚。
|
||||||
|
*/
|
||||||
|
disabled?: boolean
|
||||||
children?: MenuOptionNode[]
|
children?: MenuOptionNode[]
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -39,6 +44,8 @@ export function parsePermissionMenuItem(raw: unknown, type = ''): MenuOptionNode
|
|||||||
sort: sortRaw === null ? 0 : sortRaw,
|
sort: sortRaw === null ? 0 : sortRaw,
|
||||||
parentId: parentId === null ? null : parentId,
|
parentId: parentId === null ? null : parentId,
|
||||||
type,
|
type,
|
||||||
|
// 缺省(菜单管理页等未标记的接口)按可授予处理,保持旧行为
|
||||||
|
disabled: record.grantable === false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -202,6 +202,7 @@ onMounted(() => {
|
|||||||
<th v-if="isSuperAdmin" style="width: 140px">分组</th>
|
<th v-if="isSuperAdmin" style="width: 140px">分组</th>
|
||||||
<th style="width: 110px">来源</th>
|
<th style="width: 110px">来源</th>
|
||||||
<th style="width: 170px">创建时间</th>
|
<th style="width: 170px">创建时间</th>
|
||||||
|
<th style="width: 170px">更新时间</th>
|
||||||
<th style="width: 150px">操作</th>
|
<th style="width: 150px">操作</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
@@ -218,6 +219,7 @@ onMounted(() => {
|
|||||||
</span>
|
</span>
|
||||||
</td>
|
</td>
|
||||||
<td>{{ formatDateTime(row.createdAt) }}</td>
|
<td>{{ formatDateTime(row.createdAt) }}</td>
|
||||||
|
<td>{{ formatDateTime(row.updatedAt) }}</td>
|
||||||
<td class="ops-cell">
|
<td class="ops-cell">
|
||||||
<button class="btn btn-sm" type="button" @click="openEdit(row)">编辑</button>
|
<button class="btn btn-sm" type="button" @click="openEdit(row)">编辑</button>
|
||||||
<button class="btn btn-sm btn-danger" type="button" @click="remove(row)">删除</button>
|
<button class="btn btn-sm btn-danger" type="button" @click="remove(row)">删除</button>
|
||||||
@@ -225,10 +227,10 @@ onMounted(() => {
|
|||||||
</tr>
|
</tr>
|
||||||
</template>
|
</template>
|
||||||
<tr v-else-if="loading">
|
<tr v-else-if="loading">
|
||||||
<td :colspan="isSuperAdmin ? 7 : 6" class="empty-tip">加载中...</td>
|
<td :colspan="isSuperAdmin ? 8 : 7" class="empty-tip">加载中...</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr v-else>
|
<tr v-else>
|
||||||
<td :colspan="isSuperAdmin ? 7 : 6" class="empty-tip">暂无数据</td>
|
<td :colspan="isSuperAdmin ? 8 : 7" class="empty-tip">暂无数据</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
@@ -43,6 +43,10 @@ const lockedGroupId = computed<number | null>(() => {
|
|||||||
return groups.value.length === 1 ? groups.value[0].id : null
|
return groups.value.length === 1 ? groups.value[0].id : null
|
||||||
})
|
})
|
||||||
const jumpPage = ref('')
|
const jumpPage = ref('')
|
||||||
|
/** 顺序翻页游标:上一页响应给的 nextLastId;仅在"下一页"时使用,跳页/筛选时清空走 OFFSET。 */
|
||||||
|
const pageCursor = ref<number | null>(null)
|
||||||
|
/** 本次请求实际使用的游标(load 时决定) */
|
||||||
|
let pendingCursor: number | null = null
|
||||||
|
|
||||||
const totalPages = computed(() => Math.max(1, Math.ceil(total.value / pageSize.value)))
|
const totalPages = computed(() => Math.max(1, Math.ceil(total.value / pageSize.value)))
|
||||||
|
|
||||||
@@ -85,10 +89,15 @@ function stopExportWait(): void {
|
|||||||
async function load(): Promise<void> {
|
async function load(): Promise<void> {
|
||||||
loading.value = true
|
loading.value = true
|
||||||
try {
|
try {
|
||||||
const result = await fetchDedupeTotalList(toDedupeListParams(filter, page.value, pageSize.value))
|
const result = await fetchDedupeTotalList(
|
||||||
|
toDedupeListParams(filter, page.value, pageSize.value, pendingCursor),
|
||||||
|
)
|
||||||
rows.value = result.items
|
rows.value = result.items
|
||||||
total.value = result.total
|
total.value = result.total
|
||||||
if (result.page >= 1) page.value = result.page
|
if (result.page >= 1) page.value = result.page
|
||||||
|
// 本页响应回传的游标留作"下一页"用;空页则清空(没有更多)
|
||||||
|
pageCursor.value = result.nextLastId ?? null
|
||||||
|
pendingCursor = null
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
ElMessage.error(error instanceof Error ? error.message : '去重数据加载失败')
|
ElMessage.error(error instanceof Error ? error.message : '去重数据加载失败')
|
||||||
} finally {
|
} finally {
|
||||||
@@ -98,11 +107,15 @@ async function load(): Promise<void> {
|
|||||||
|
|
||||||
function apply(): void {
|
function apply(): void {
|
||||||
page.value = 1
|
page.value = 1
|
||||||
|
pageCursor.value = null
|
||||||
|
pendingCursor = null
|
||||||
void load()
|
void load()
|
||||||
}
|
}
|
||||||
|
|
||||||
function changePage(next: number): void {
|
function changePage(next: number): void {
|
||||||
if (next < 1 || next > totalPages.value) return
|
if (next < 1 || next > totalPages.value) return
|
||||||
|
// 只有"顺序下一页"用 keyset 游标(避免深分页 offset);跳页/回退走 OFFSET,行为不变
|
||||||
|
pendingCursor = next === page.value + 1 ? pageCursor.value : null
|
||||||
page.value = next
|
page.value = next
|
||||||
void load()
|
void load()
|
||||||
}
|
}
|
||||||
@@ -119,6 +132,8 @@ function goJump(): void {
|
|||||||
function changeSize(size: number) {
|
function changeSize(size: number) {
|
||||||
pageSize.value = size
|
pageSize.value = size
|
||||||
page.value = 1
|
page.value = 1
|
||||||
|
pageCursor.value = null
|
||||||
|
pendingCursor = null
|
||||||
void load()
|
void load()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -360,6 +375,7 @@ onMounted(() => {
|
|||||||
<th>用户名</th>
|
<th>用户名</th>
|
||||||
<th v-if="isSuperAdmin">分组</th>
|
<th v-if="isSuperAdmin">分组</th>
|
||||||
<th>创建时间</th>
|
<th>创建时间</th>
|
||||||
|
<th>更新时间</th>
|
||||||
<th>操作</th>
|
<th>操作</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
@@ -372,6 +388,7 @@ onMounted(() => {
|
|||||||
<td>{{ row.username || '-' }}</td>
|
<td>{{ row.username || '-' }}</td>
|
||||||
<td v-if="isSuperAdmin">{{ row.groupName || '未分组' }}</td>
|
<td v-if="isSuperAdmin">{{ row.groupName || '未分组' }}</td>
|
||||||
<td>{{ formatDateTime(row.createdAt) }}</td>
|
<td>{{ formatDateTime(row.createdAt) }}</td>
|
||||||
|
<td>{{ formatDateTime(row.updatedAt) }}</td>
|
||||||
<td class="ops-cell">
|
<td class="ops-cell">
|
||||||
<button class="btn btn-sm" type="button" @click="openEdit(row)">编辑</button>
|
<button class="btn btn-sm" type="button" @click="openEdit(row)">编辑</button>
|
||||||
<button class="btn btn-sm btn-danger" type="button" @click="removeRow(row)">删除</button>
|
<button class="btn btn-sm btn-danger" type="button" @click="removeRow(row)">删除</button>
|
||||||
@@ -379,10 +396,10 @@ onMounted(() => {
|
|||||||
</tr>
|
</tr>
|
||||||
</template>
|
</template>
|
||||||
<tr v-else-if="loading">
|
<tr v-else-if="loading">
|
||||||
<td :colspan="isSuperAdmin ? 7 : 6" class="empty-tip">加载中...</td>
|
<td :colspan="isSuperAdmin ? 8 : 7" class="empty-tip">加载中...</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr v-else>
|
<tr v-else>
|
||||||
<td :colspan="isSuperAdmin ? 7 : 6" class="empty-tip">暂无总数据</td>
|
<td :colspan="isSuperAdmin ? 8 : 7" class="empty-tip">暂无总数据</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import {
|
|||||||
} from './product-category-model.ts'
|
} from './product-category-model.ts'
|
||||||
import { categoryDraftError, createProductCategoryDraft, toCategorySaveRequest, type ProductCategoryDraft } from './product-category-editor-model.ts'
|
import { categoryDraftError, createProductCategoryDraft, toCategorySaveRequest, type ProductCategoryDraft } from './product-category-editor-model.ts'
|
||||||
import { buildProductCategoryExportUrl } from './product-category-export.ts'
|
import { buildProductCategoryExportUrl } from './product-category-export.ts'
|
||||||
|
import { formatDateTime } from '@/utils/datetime'
|
||||||
import OldPagination from '@/components/OldPagination.vue'
|
import OldPagination from '@/components/OldPagination.vue'
|
||||||
|
|
||||||
const loading = ref(false)
|
const loading = ref(false)
|
||||||
@@ -330,6 +331,8 @@ onMounted(loadTree)
|
|||||||
<th style="width: 90px">排序</th>
|
<th style="width: 90px">排序</th>
|
||||||
<th style="width: 120px">来源</th>
|
<th style="width: 120px">来源</th>
|
||||||
<th>备注</th>
|
<th>备注</th>
|
||||||
|
<th style="width: 170px">创建时间</th>
|
||||||
|
<th style="width: 170px">更新时间</th>
|
||||||
<th style="width: 150px">操作</th>
|
<th style="width: 150px">操作</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
@@ -337,7 +340,7 @@ onMounted(loadTree)
|
|||||||
<template v-if="treeRows.length">
|
<template v-if="treeRows.length">
|
||||||
<template v-for="row in treeRows" :key="String(row.node.id)">
|
<template v-for="row in treeRows" :key="String(row.node.id)">
|
||||||
<tr v-if="isCategoryLoadMoreNode(row.node)" class="load-more-row">
|
<tr v-if="isCategoryLoadMoreNode(row.node)" class="load-more-row">
|
||||||
<td colspan="6">
|
<td colspan="8">
|
||||||
<div class="load-more-cell">
|
<div class="load-more-cell">
|
||||||
<span class="tree-indent" :style="{ width: `${(row.depth + 1) * 24}px` }"></span>
|
<span class="tree-indent" :style="{ width: `${(row.depth + 1) * 24}px` }"></span>
|
||||||
<button class="btn btn-sm btn-secondary" type="button" :disabled="moreLoading(row.node)" @click="loadMoreChildren(row.node)">
|
<button class="btn btn-sm btn-secondary" type="button" :disabled="moreLoading(row.node)" @click="loadMoreChildren(row.node)">
|
||||||
@@ -376,6 +379,8 @@ onMounted(loadTree)
|
|||||||
<span v-if="row.node.description" class="node-desc" :title="row.node.description">{{ row.node.description }}</span>
|
<span v-if="row.node.description" class="node-desc" :title="row.node.description">{{ row.node.description }}</span>
|
||||||
<span v-else class="dim">—</span>
|
<span v-else class="dim">—</span>
|
||||||
</td>
|
</td>
|
||||||
|
<td>{{ formatDateTime(row.node.createdAt) }}</td>
|
||||||
|
<td>{{ formatDateTime(row.node.updatedAt) }}</td>
|
||||||
<td class="ops-cell">
|
<td class="ops-cell">
|
||||||
<button class="btn btn-sm" type="button" @click="openEdit(row.node)">编辑</button>
|
<button class="btn btn-sm" type="button" @click="openEdit(row.node)">编辑</button>
|
||||||
<button
|
<button
|
||||||
@@ -392,10 +397,10 @@ onMounted(loadTree)
|
|||||||
</template>
|
</template>
|
||||||
</template>
|
</template>
|
||||||
<tr v-else-if="loading">
|
<tr v-else-if="loading">
|
||||||
<td colspan="6" class="empty-tip">加载中...</td>
|
<td colspan="8" class="empty-tip">加载中...</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr v-else>
|
<tr v-else>
|
||||||
<td colspan="6" class="empty-tip">暂无商品类目</td>
|
<td colspan="8" class="empty-tip">暂无商品类目</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
<tbody v-else>
|
<tbody v-else>
|
||||||
@@ -411,6 +416,8 @@ onMounted(loadTree)
|
|||||||
<span v-if="row.description" class="node-desc" :title="row.description">{{ row.description }}</span>
|
<span v-if="row.description" class="node-desc" :title="row.description">{{ row.description }}</span>
|
||||||
<span v-else class="dim">—</span>
|
<span v-else class="dim">—</span>
|
||||||
</td>
|
</td>
|
||||||
|
<td>{{ formatDateTime(row.createdAt) }}</td>
|
||||||
|
<td>{{ formatDateTime(row.updatedAt) }}</td>
|
||||||
<td class="ops-cell">
|
<td class="ops-cell">
|
||||||
<button class="btn btn-sm" type="button" @click="openEdit(row)">编辑</button>
|
<button class="btn btn-sm" type="button" @click="openEdit(row)">编辑</button>
|
||||||
<button
|
<button
|
||||||
@@ -426,10 +433,10 @@ onMounted(loadTree)
|
|||||||
</tr>
|
</tr>
|
||||||
</template>
|
</template>
|
||||||
<tr v-else-if="searchLoading">
|
<tr v-else-if="searchLoading">
|
||||||
<td colspan="6" class="empty-tip">搜索中...</td>
|
<td colspan="8" class="empty-tip">搜索中...</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr v-else>
|
<tr v-else>
|
||||||
<td colspan="6" class="empty-tip">暂无搜索结果</td>
|
<td colspan="8" class="empty-tip">暂无搜索结果</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
import { computed, onMounted, reactive, ref } from 'vue'
|
import { computed, onMounted, reactive, ref } from 'vue'
|
||||||
import { ElMessage, ElMessageBox } from 'element-plus'
|
import { ElMessage, ElMessageBox } from 'element-plus'
|
||||||
import CopyText from '@/components/CopyText.vue'
|
import CopyText from '@/components/CopyText.vue'
|
||||||
|
import { formatDateTime } from '@/utils/datetime'
|
||||||
import { createQueryAsin, fetchQueryAsinList, fetchShopNamesByGroup } from './query-asin-api.ts'
|
import { createQueryAsin, fetchQueryAsinList, fetchShopNamesByGroup } from './query-asin-api.ts'
|
||||||
import { QUERY_ASIN_COUNTRIES, queryAsinDisplayRows, type QueryAsinItem } from './query-asin-model.ts'
|
import { QUERY_ASIN_COUNTRIES, queryAsinDisplayRows, type QueryAsinItem } from './query-asin-model.ts'
|
||||||
import { asinCountryLabel } from './asin-country.ts'
|
import { asinCountryLabel } from './asin-country.ts'
|
||||||
@@ -394,6 +395,8 @@ onMounted(() => {
|
|||||||
<th style="width: 15%">店铺名</th>
|
<th style="width: 15%">店铺名</th>
|
||||||
<th style="width: 24%">ASIN</th>
|
<th style="width: 24%">ASIN</th>
|
||||||
<th style="width: 12%">国家</th>
|
<th style="width: 12%">国家</th>
|
||||||
|
<th style="width: 170px">创建时间</th>
|
||||||
|
<th style="width: 170px">更新时间</th>
|
||||||
<th style="width: 9%">操作</th>
|
<th style="width: 9%">操作</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
@@ -413,6 +416,8 @@ onMounted(() => {
|
|||||||
</td>
|
</td>
|
||||||
<td>{{ asinCountryLabel(row.country || '') }}</td>
|
<td>{{ asinCountryLabel(row.country || '') }}</td>
|
||||||
<template v-if="row.isFirst">
|
<template v-if="row.isFirst">
|
||||||
|
<td :rowspan="row.rowspan">{{ formatDateTime(row.item.createdAt) }}</td>
|
||||||
|
<td :rowspan="row.rowspan">{{ formatDateTime(row.item.updatedAt) }}</td>
|
||||||
<td :rowspan="row.rowspan" class="asin-col-actions">
|
<td :rowspan="row.rowspan" class="asin-col-actions">
|
||||||
<button class="btn btn-sm" type="button" @click="openConfig(row.item as QueryAsinItem)">配置</button>
|
<button class="btn btn-sm" type="button" @click="openConfig(row.item as QueryAsinItem)">配置</button>
|
||||||
<button class="btn btn-sm btn-danger" type="button" @click="removeShopAsin(row.item as QueryAsinItem)">删除</button>
|
<button class="btn btn-sm btn-danger" type="button" @click="removeShopAsin(row.item as QueryAsinItem)">删除</button>
|
||||||
@@ -421,10 +426,10 @@ onMounted(() => {
|
|||||||
</tr>
|
</tr>
|
||||||
</template>
|
</template>
|
||||||
<tr v-else-if="loading">
|
<tr v-else-if="loading">
|
||||||
<td :colspan="isSuperAdmin ? 6 : 5" class="empty-tip">加载中...</td>
|
<td :colspan="isSuperAdmin ? 8 : 7" class="empty-tip">加载中...</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr v-else>
|
<tr v-else>
|
||||||
<td :colspan="isSuperAdmin ? 6 : 5" class="empty-tip">暂无数据</td>
|
<td :colspan="isSuperAdmin ? 8 : 7" class="empty-tip">暂无数据</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
import { computed, onMounted, reactive, ref } from 'vue'
|
import { computed, onMounted, reactive, ref } from 'vue'
|
||||||
import { ElMessage, ElMessageBox } from 'element-plus'
|
import { ElMessage, ElMessageBox } from 'element-plus'
|
||||||
import CopyText from '@/components/CopyText.vue'
|
import CopyText from '@/components/CopyText.vue'
|
||||||
|
import { formatDateTime } from '@/utils/datetime'
|
||||||
import { createSkipPriceAsin, fetchSkipPriceList } from './skip-price-api.ts'
|
import { createSkipPriceAsin, fetchSkipPriceList } from './skip-price-api.ts'
|
||||||
import { skipPriceDisplayRows, type SkipPriceItem } from './skip-price-model.ts'
|
import { skipPriceDisplayRows, type SkipPriceItem } from './skip-price-model.ts'
|
||||||
import { asinCountryLabel } from './asin-country.ts'
|
import { asinCountryLabel } from './asin-country.ts'
|
||||||
@@ -447,6 +448,8 @@ onMounted(() => {
|
|||||||
<th style="width: 24%">ASIN</th>
|
<th style="width: 24%">ASIN</th>
|
||||||
<th style="width: 12%">国家</th>
|
<th style="width: 12%">国家</th>
|
||||||
<th style="width: 10%">最低价</th>
|
<th style="width: 10%">最低价</th>
|
||||||
|
<th style="width: 170px">创建时间</th>
|
||||||
|
<th style="width: 170px">更新时间</th>
|
||||||
<th style="width: 9%">操作</th>
|
<th style="width: 9%">操作</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
@@ -467,6 +470,8 @@ onMounted(() => {
|
|||||||
<td>{{ asinCountryLabel(row.country || '') }}</td>
|
<td>{{ asinCountryLabel(row.country || '') }}</td>
|
||||||
<td class="price-cell">{{ row.minimumPrice !== '' ? row.minimumPrice : '-' }}</td>
|
<td class="price-cell">{{ row.minimumPrice !== '' ? row.minimumPrice : '-' }}</td>
|
||||||
<template v-if="row.isFirst">
|
<template v-if="row.isFirst">
|
||||||
|
<td :rowspan="row.rowspan">{{ formatDateTime(row.item.createdAt) }}</td>
|
||||||
|
<td :rowspan="row.rowspan">{{ formatDateTime(row.item.updatedAt) }}</td>
|
||||||
<td :rowspan="row.rowspan" class="asin-col-actions">
|
<td :rowspan="row.rowspan" class="asin-col-actions">
|
||||||
<button class="btn btn-sm" type="button" @click="openConfig(row.item as SkipPriceItem)">配置</button>
|
<button class="btn btn-sm" type="button" @click="openConfig(row.item as SkipPriceItem)">配置</button>
|
||||||
<button class="btn btn-sm btn-danger" type="button" @click="removeRow(row.item as SkipPriceItem)">删除</button>
|
<button class="btn btn-sm btn-danger" type="button" @click="removeRow(row.item as SkipPriceItem)">删除</button>
|
||||||
@@ -475,10 +480,10 @@ onMounted(() => {
|
|||||||
</tr>
|
</tr>
|
||||||
</template>
|
</template>
|
||||||
<tr v-else-if="loading">
|
<tr v-else-if="loading">
|
||||||
<td :colspan="isSuperAdmin ? 7 : 6" class="empty-tip">加载中...</td>
|
<td :colspan="isSuperAdmin ? 9 : 8" class="empty-tip">加载中...</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr v-else>
|
<tr v-else>
|
||||||
<td :colspan="isSuperAdmin ? 7 : 6" class="empty-tip">暂无数据</td>
|
<td :colspan="isSuperAdmin ? 9 : 8" class="empty-tip">暂无数据</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
@@ -24,6 +24,8 @@ export interface AsinListParams {
|
|||||||
groupId?: number | null
|
groupId?: number | null
|
||||||
/** 国家代码(如 DE、UK)。 */
|
/** 国家代码(如 DE、UK)。 */
|
||||||
country?: string
|
country?: string
|
||||||
|
/** 顺序翻页游标(上一页返回的 nextLastId):传了就忽略 page 偏移,走 keyset。 */
|
||||||
|
lastId?: number
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 序列化到 Java 分页接口的查询参数(snake_case)。 */
|
/** 序列化到 Java 分页接口的查询参数(snake_case)。 */
|
||||||
@@ -36,6 +38,8 @@ export interface AsinPageQuery {
|
|||||||
end_date?: string
|
end_date?: string
|
||||||
group_id?: number
|
group_id?: number
|
||||||
country?: string
|
country?: string
|
||||||
|
/** 顺序翻页游标(keyset):传了就忽略 page 偏移。 */
|
||||||
|
last_id?: number
|
||||||
}
|
}
|
||||||
|
|
||||||
function finiteInt(value: unknown): number | null {
|
function finiteInt(value: unknown): number | null {
|
||||||
@@ -77,5 +81,6 @@ export function toAsinPageQuery(params: AsinListParams): AsinPageQuery {
|
|||||||
if (params.endDate) query.end_date = params.endDate
|
if (params.endDate) query.end_date = params.endDate
|
||||||
if (params.groupId != null) query.group_id = params.groupId
|
if (params.groupId != null) query.group_id = params.groupId
|
||||||
if (params.country) query.country = params.country
|
if (params.country) query.country = params.country
|
||||||
|
if (typeof params.lastId === 'number' && params.lastId > 0) query.last_id = params.lastId
|
||||||
return query
|
return query
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,8 +30,10 @@ export function toDedupeListParams(
|
|||||||
state: DedupeTotalFilterState,
|
state: DedupeTotalFilterState,
|
||||||
page: number,
|
page: number,
|
||||||
pageSize: number,
|
pageSize: number,
|
||||||
|
lastId?: number | null,
|
||||||
): AsinListParams {
|
): AsinListParams {
|
||||||
const params: AsinListParams = { page, pageSize }
|
const params: AsinListParams = { page, pageSize }
|
||||||
|
if (typeof lastId === 'number' && lastId > 0) params.lastId = lastId
|
||||||
const keyword = (state.keyword || '').trim()
|
const keyword = (state.keyword || '').trim()
|
||||||
const username = (state.username || '').trim()
|
const username = (state.username || '').trim()
|
||||||
const country = (state.country || '').trim()
|
const country = (state.country || '').trim()
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ export interface DedupeTotalItem {
|
|||||||
uploaderUserId: number | null
|
uploaderUserId: number | null
|
||||||
username: string
|
username: string
|
||||||
createdAt?: string
|
createdAt?: string
|
||||||
|
updatedAt?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface DedupeTotalPageResult {
|
export interface DedupeTotalPageResult {
|
||||||
@@ -18,6 +19,8 @@ export interface DedupeTotalPageResult {
|
|||||||
total: number
|
total: number
|
||||||
page: number
|
page: number
|
||||||
pageSize: number
|
pageSize: number
|
||||||
|
/** 顺序翻页游标:本页最后一行 id;下一页回传它即可走 keyset。 */
|
||||||
|
nextLastId?: number
|
||||||
}
|
}
|
||||||
|
|
||||||
export function emptyDedupeTotalPage(): DedupeTotalPageResult {
|
export function emptyDedupeTotalPage(): DedupeTotalPageResult {
|
||||||
@@ -49,6 +52,8 @@ export function toDedupeTotalItem(raw: unknown): DedupeTotalItem | null {
|
|||||||
}
|
}
|
||||||
const createdAt = text(record.createdAt ?? record.created_at)
|
const createdAt = text(record.createdAt ?? record.created_at)
|
||||||
if (createdAt) item.createdAt = createdAt
|
if (createdAt) item.createdAt = createdAt
|
||||||
|
const updatedAt = text(record.updatedAt ?? record.updated_at)
|
||||||
|
if (updatedAt) item.updatedAt = updatedAt
|
||||||
return item
|
return item
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -65,6 +70,8 @@ export function parseDedupeTotalPage(payload: unknown): DedupeTotalPageResult {
|
|||||||
}
|
}
|
||||||
if (typeof record.total === 'number') out.total = Math.floor(record.total)
|
if (typeof record.total === 'number') out.total = Math.floor(record.total)
|
||||||
if (typeof record.page === 'number' && record.page >= 1) out.page = Math.floor(record.page)
|
if (typeof record.page === 'number' && record.page >= 1) out.page = Math.floor(record.page)
|
||||||
|
const rawNextLastId = record.nextLastId ?? record.next_last_id
|
||||||
|
if (typeof rawNextLastId === 'number' && rawNextLastId > 0) out.nextLastId = Math.floor(rawNextLastId)
|
||||||
const rawSize = record.pageSize ?? record.page_size
|
const rawSize = record.pageSize ?? record.page_size
|
||||||
if (typeof rawSize === 'number' && rawSize >= 1) out.pageSize = Math.floor(rawSize)
|
if (typeof rawSize === 'number' && rawSize >= 1) out.pageSize = Math.floor(rawSize)
|
||||||
return out
|
return out
|
||||||
|
|||||||
@@ -14,6 +14,8 @@ export interface ProductCategoryNode {
|
|||||||
childCount: number
|
childCount: number
|
||||||
level: number | null
|
level: number | null
|
||||||
path: string
|
path: string
|
||||||
|
createdAt?: string
|
||||||
|
updatedAt?: string
|
||||||
children: ProductCategoryNode[]
|
children: ProductCategoryNode[]
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -72,6 +74,10 @@ export function toProductCategoryNode(raw: unknown): ProductCategoryNode | null
|
|||||||
path: text(record.path),
|
path: text(record.path),
|
||||||
children: [],
|
children: [],
|
||||||
}
|
}
|
||||||
|
const createdAt = text(record.createdAt ?? record.created_at)
|
||||||
|
if (createdAt) node.createdAt = createdAt
|
||||||
|
const updatedAt = text(record.updatedAt ?? record.updated_at)
|
||||||
|
if (updatedAt) node.updatedAt = updatedAt
|
||||||
if (Array.isArray(record.children)) {
|
if (Array.isArray(record.children)) {
|
||||||
node.children = record.children
|
node.children = record.children
|
||||||
.map((child) => toProductCategoryNode(child))
|
.map((child) => toProductCategoryNode(child))
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ export interface InvalidAsinItem {
|
|||||||
groupName: string
|
groupName: string
|
||||||
recordSource: string
|
recordSource: string
|
||||||
createdAt?: string
|
createdAt?: string
|
||||||
|
updatedAt?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface InvalidAsinPageResult {
|
export interface InvalidAsinPageResult {
|
||||||
@@ -74,6 +75,8 @@ export function toInvalidAsinItem(raw: unknown): InvalidAsinItem | null {
|
|||||||
}
|
}
|
||||||
const createdAt = text(record.createdAt ?? record.created_at)
|
const createdAt = text(record.createdAt ?? record.created_at)
|
||||||
if (createdAt) item.createdAt = createdAt
|
if (createdAt) item.createdAt = createdAt
|
||||||
|
const updatedAt = text(record.updatedAt ?? record.updated_at)
|
||||||
|
if (updatedAt) item.updatedAt = updatedAt
|
||||||
return item
|
return item
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,719 @@
|
|||||||
|
<script setup lang="ts">
|
||||||
|
/** 日志管理页:桌面客户端 / 麦象采集机上报的日志浏览(仅超管)。
|
||||||
|
* 支持来源/日期/关键字筛选,尾部内容查看(自动滚到底、可向前加载更早内容)、
|
||||||
|
* 完整下载、删除;「采集配置」可调全量/精选模式(全局默认 + 终端覆盖)。 */
|
||||||
|
import { computed, nextTick, onMounted, ref } from 'vue'
|
||||||
|
import OldPagination from '@/components/OldPagination.vue'
|
||||||
|
import { ElMessage } from 'element-plus'
|
||||||
|
import { formatDateTime } from '@/utils/datetime'
|
||||||
|
import {
|
||||||
|
deleteDeviceLogFile,
|
||||||
|
deleteDeviceLogOverride,
|
||||||
|
deviceLogDownloadUrl,
|
||||||
|
fetchDeviceLogConfig,
|
||||||
|
fetchDeviceLogContent,
|
||||||
|
fetchDeviceLogDevices,
|
||||||
|
fetchDeviceLogFiles,
|
||||||
|
updateDeviceLogGlobalMode,
|
||||||
|
updateDeviceLogOverride,
|
||||||
|
type DeviceLogContent,
|
||||||
|
type DeviceLogDevice,
|
||||||
|
type DeviceLogFileRow,
|
||||||
|
type DeviceLogOverride,
|
||||||
|
} from '@/api/device-logs'
|
||||||
|
|
||||||
|
const SOURCE_OPTIONS = [
|
||||||
|
{ value: '', label: '全部来源' },
|
||||||
|
{ value: 'client', label: '桌面客户端' },
|
||||||
|
{ value: 'maixiang', label: '麦象采集机' },
|
||||||
|
]
|
||||||
|
|
||||||
|
function sourceLabel(source: string): string {
|
||||||
|
if (source === 'client') return '桌面客户端'
|
||||||
|
if (source === 'maixiang') return '麦象采集机'
|
||||||
|
return source || '—'
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatBytes(bytes: number | null | undefined): string {
|
||||||
|
if (bytes == null || !Number.isFinite(bytes) || bytes <= 0) return '—'
|
||||||
|
if (bytes < 1024) return `${bytes} B`
|
||||||
|
const units = ['KB', 'MB', 'GB']
|
||||||
|
let value = bytes / 1024
|
||||||
|
let unitIndex = 0
|
||||||
|
while (value >= 1024 && unitIndex < units.length - 1) {
|
||||||
|
value /= 1024
|
||||||
|
unitIndex += 1
|
||||||
|
}
|
||||||
|
return `${value >= 100 ? value.toFixed(0) : value.toFixed(1)} ${units[unitIndex]}`
|
||||||
|
}
|
||||||
|
|
||||||
|
const loading = ref(false)
|
||||||
|
const rows = ref<DeviceLogFileRow[]>([])
|
||||||
|
const total = ref(0)
|
||||||
|
const page = ref(1)
|
||||||
|
const pageSize = ref(15)
|
||||||
|
const sourceFilter = ref('')
|
||||||
|
const keyword = ref('')
|
||||||
|
const startDate = ref('')
|
||||||
|
const endDate = ref('')
|
||||||
|
const retentionDays = ref(7)
|
||||||
|
|
||||||
|
const totalPages = computed(() => Math.max(1, Math.ceil(total.value / pageSize.value)))
|
||||||
|
|
||||||
|
async function load() {
|
||||||
|
loading.value = true
|
||||||
|
try {
|
||||||
|
const result = await fetchDeviceLogFiles({
|
||||||
|
source: sourceFilter.value || undefined,
|
||||||
|
keyword: keyword.value.trim() || undefined,
|
||||||
|
startDate: startDate.value || undefined,
|
||||||
|
endDate: endDate.value || undefined,
|
||||||
|
page: page.value,
|
||||||
|
pageSize: pageSize.value,
|
||||||
|
})
|
||||||
|
rows.value = result?.items || []
|
||||||
|
total.value = Number(result?.total || 0)
|
||||||
|
if (result?.retentionDays) retentionDays.value = result.retentionDays
|
||||||
|
} catch (error) {
|
||||||
|
ElMessage.error(error instanceof Error ? error.message : '日志列表加载失败')
|
||||||
|
} finally {
|
||||||
|
loading.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function search() {
|
||||||
|
page.value = 1
|
||||||
|
load()
|
||||||
|
}
|
||||||
|
|
||||||
|
function reset() {
|
||||||
|
sourceFilter.value = ''
|
||||||
|
keyword.value = ''
|
||||||
|
startDate.value = ''
|
||||||
|
endDate.value = ''
|
||||||
|
page.value = 1
|
||||||
|
load()
|
||||||
|
}
|
||||||
|
|
||||||
|
function changePage(next: number) {
|
||||||
|
if (next < 1 || next > totalPages.value) return
|
||||||
|
page.value = next
|
||||||
|
load()
|
||||||
|
}
|
||||||
|
|
||||||
|
function changeSize(size: number) {
|
||||||
|
pageSize.value = size
|
||||||
|
page.value = 1
|
||||||
|
load()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------- 内容查看
|
||||||
|
|
||||||
|
const TAIL_STEP = 256 * 1024
|
||||||
|
const TAIL_MAX = 8 * 1024 * 1024
|
||||||
|
|
||||||
|
const viewerVisible = ref(false)
|
||||||
|
const viewerLoading = ref(false)
|
||||||
|
const viewerFile = ref<DeviceLogFileRow | null>(null)
|
||||||
|
const viewerContent = ref<DeviceLogContent | null>(null)
|
||||||
|
const viewerMaxBytes = ref(TAIL_STEP)
|
||||||
|
const viewerPre = ref<HTMLElement | null>(null)
|
||||||
|
|
||||||
|
async function openViewer(row: DeviceLogFileRow) {
|
||||||
|
viewerFile.value = row
|
||||||
|
viewerMaxBytes.value = TAIL_STEP
|
||||||
|
viewerContent.value = null
|
||||||
|
viewerVisible.value = true
|
||||||
|
await loadContent(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadContent(scrollToBottom: boolean) {
|
||||||
|
if (!viewerFile.value) return
|
||||||
|
viewerLoading.value = true
|
||||||
|
try {
|
||||||
|
viewerContent.value = await fetchDeviceLogContent(viewerFile.value.id, viewerMaxBytes.value)
|
||||||
|
if (scrollToBottom) {
|
||||||
|
await nextTick()
|
||||||
|
if (viewerPre.value) viewerPre.value.scrollTop = viewerPre.value.scrollHeight
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
ElMessage.error(error instanceof Error ? error.message : '日志内容加载失败')
|
||||||
|
} finally {
|
||||||
|
viewerLoading.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadMore() {
|
||||||
|
viewerMaxBytes.value = Math.min(viewerMaxBytes.value * 2, TAIL_MAX)
|
||||||
|
loadContent(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function remove(row: DeviceLogFileRow) {
|
||||||
|
if (!window.confirm(`确认删除「${row.fileName}」(${row.deviceName || row.deviceId})的云端日志?删除后不可恢复。`)) return
|
||||||
|
try {
|
||||||
|
await deleteDeviceLogFile(row.id)
|
||||||
|
ElMessage.success('已删除')
|
||||||
|
load()
|
||||||
|
} catch (error) {
|
||||||
|
ElMessage.error(error instanceof Error ? error.message : '删除失败')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------- 采集配置
|
||||||
|
|
||||||
|
const configVisible = ref(false)
|
||||||
|
const configLoading = ref(false)
|
||||||
|
const globalMode = ref('full')
|
||||||
|
const overrides = ref<DeviceLogOverride[]>([])
|
||||||
|
const devices = ref<DeviceLogDevice[]>([])
|
||||||
|
const newOverrideKey = ref('')
|
||||||
|
const newOverrideMode = ref('selected')
|
||||||
|
|
||||||
|
const MODE_OPTIONS = [
|
||||||
|
{ value: 'full', label: '全量采集' },
|
||||||
|
{ value: 'selected', label: '精选采集' },
|
||||||
|
]
|
||||||
|
|
||||||
|
function modeLabel(mode: string): string {
|
||||||
|
return mode === 'selected' ? '精选' : '全量'
|
||||||
|
}
|
||||||
|
|
||||||
|
async function openConfig() {
|
||||||
|
configVisible.value = true
|
||||||
|
await loadConfig()
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadConfig() {
|
||||||
|
configLoading.value = true
|
||||||
|
try {
|
||||||
|
const data = await fetchDeviceLogConfig()
|
||||||
|
globalMode.value = data.globalMode || 'full'
|
||||||
|
overrides.value = data.overrides || []
|
||||||
|
devices.value = (await fetchDeviceLogDevices()) || []
|
||||||
|
} catch (error) {
|
||||||
|
ElMessage.error(error instanceof Error ? error.message : '采集配置加载失败')
|
||||||
|
} finally {
|
||||||
|
configLoading.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function saveGlobalMode(mode: string) {
|
||||||
|
if (mode === globalMode.value) return
|
||||||
|
try {
|
||||||
|
await updateDeviceLogGlobalMode(mode)
|
||||||
|
globalMode.value = mode
|
||||||
|
ElMessage.success(`全局采集模式已切换为「${modeLabel(mode)}」`)
|
||||||
|
} catch (error) {
|
||||||
|
ElMessage.error(error instanceof Error ? error.message : '保存失败')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function addOverride() {
|
||||||
|
if (!newOverrideKey.value) {
|
||||||
|
ElMessage.warning('请先选择终端')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
const [source, deviceId] = newOverrideKey.value.split('|')
|
||||||
|
const device = devices.value.find((item) => item.source === source && item.deviceId === deviceId)
|
||||||
|
try {
|
||||||
|
await updateDeviceLogOverride(source, deviceId, device?.deviceName || null, newOverrideMode.value)
|
||||||
|
ElMessage.success('终端覆盖已保存')
|
||||||
|
newOverrideKey.value = ''
|
||||||
|
await loadConfig()
|
||||||
|
} catch (error) {
|
||||||
|
ElMessage.error(error instanceof Error ? error.message : '保存失败')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function removeOverride(row: DeviceLogOverride) {
|
||||||
|
const who = row.deviceName || row.deviceId
|
||||||
|
if (!window.confirm(`确认删除终端「${who}」的采集覆盖(回落到全局默认)?`)) return
|
||||||
|
try {
|
||||||
|
await deleteDeviceLogOverride(row.id)
|
||||||
|
ElMessage.success('已删除')
|
||||||
|
await loadConfig()
|
||||||
|
} catch (error) {
|
||||||
|
ElMessage.error(error instanceof Error ? error.message : '删除失败')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
onMounted(load)
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<div class="device-logs-view">
|
||||||
|
<section class="panel-box">
|
||||||
|
<div class="logs-head">
|
||||||
|
<h3>日志文件列表</h3>
|
||||||
|
<div class="logs-head-tools">
|
||||||
|
<span class="retention-tip">云端仅保留 {{ retentionDays }} 天</span>
|
||||||
|
<button class="btn btn-ghost" type="button" @click="openConfig">采集配置</button>
|
||||||
|
<button class="btn" type="button" @click="load">刷新</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="form-row logs-filter-row">
|
||||||
|
<div class="form-group" style="min-width: 150px">
|
||||||
|
<label>来源</label>
|
||||||
|
<select v-model="sourceFilter">
|
||||||
|
<option v-for="option in SOURCE_OPTIONS" :key="option.value" :value="option.value">{{ option.label }}</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
<div class="form-group" style="min-width: 220px">
|
||||||
|
<label>设备 / 文件名</label>
|
||||||
|
<input v-model="keyword" type="text" placeholder="模糊搜索设备名、设备ID、文件名" @keyup.enter="search" />
|
||||||
|
</div>
|
||||||
|
<div class="form-group" style="min-width: 150px">
|
||||||
|
<label>日志日期(起)</label>
|
||||||
|
<input v-model="startDate" type="date" />
|
||||||
|
</div>
|
||||||
|
<div class="form-group" style="min-width: 150px">
|
||||||
|
<label>日志日期(止)</label>
|
||||||
|
<input v-model="endDate" type="date" />
|
||||||
|
</div>
|
||||||
|
<div class="form-group">
|
||||||
|
<label> </label>
|
||||||
|
<div class="filter-actions">
|
||||||
|
<button class="btn" type="button" @click="search">查询</button>
|
||||||
|
<button class="btn btn-ghost" type="button" @click="reset">重置</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="logs-table-scroll">
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th style="width: 110px">来源</th>
|
||||||
|
<th style="width: 230px">设备</th>
|
||||||
|
<th style="width: 130px">用户</th>
|
||||||
|
<th style="width: 230px">文件名</th>
|
||||||
|
<th style="width: 110px">日志日期</th>
|
||||||
|
<th style="width: 100px">已收大小</th>
|
||||||
|
<th style="width: 80px">片段数</th>
|
||||||
|
<th style="width: 170px">最后更新</th>
|
||||||
|
<th style="width: 190px">操作</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<template v-if="rows.length">
|
||||||
|
<tr v-for="row in rows" :key="row.id">
|
||||||
|
<td>
|
||||||
|
<span class="source-pill" :class="row.source === 'maixiang' ? 'is-maixiang' : 'is-client'">
|
||||||
|
{{ sourceLabel(row.source) }}
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<span class="device-name" :title="row.deviceId">{{ row.deviceName || '—' }}</span>
|
||||||
|
<span class="device-id" :title="row.deviceId">{{ row.deviceId }}</span>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<span v-if="row.username" class="user-name">{{ row.username }}</span>
|
||||||
|
<span v-else-if="row.uid" class="user-name">UID {{ row.uid }}</span>
|
||||||
|
<span v-else class="dim">—</span>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<span class="file-name" :title="row.fileName">{{ row.fileName }}</span>
|
||||||
|
</td>
|
||||||
|
<td>{{ row.logDate }}</td>
|
||||||
|
<td>{{ formatBytes(row.uploadedBytes) }}</td>
|
||||||
|
<td>{{ row.partCount ?? 0 }}</td>
|
||||||
|
<td>{{ row.lastUploadAt ? formatDateTime(row.lastUploadAt) : '—' }}</td>
|
||||||
|
<td class="ops-cell">
|
||||||
|
<button class="btn btn-sm" type="button" @click="openViewer(row)">查看</button>
|
||||||
|
<a class="btn btn-sm btn-ghost dl-link" :href="deviceLogDownloadUrl(row.id)" download>下载</a>
|
||||||
|
<button class="btn btn-sm btn-danger" type="button" @click="remove(row)">删除</button>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</template>
|
||||||
|
<tr v-else-if="loading">
|
||||||
|
<td colspan="9" class="empty-tip">加载中...</td>
|
||||||
|
</tr>
|
||||||
|
<tr v-else>
|
||||||
|
<td colspan="9" class="empty-tip">
|
||||||
|
{{ keyword || sourceFilter || startDate || endDate ? '暂无匹配日志' : '暂无日志上报(客户端/采集机上报后自动出现在这里)' }}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
<OldPagination :total="total" :page="page" :page-size="pageSize" @change="changePage" @size-change="changeSize" />
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<el-dialog v-model="viewerVisible" :title="viewerFile ? `${viewerFile.fileName}(${viewerFile.deviceName || viewerFile.deviceId})` : '日志内容'" width="900px" top="5vh">
|
||||||
|
<div class="viewer-toolbar">
|
||||||
|
<span class="viewer-meta">
|
||||||
|
云端已收 {{ formatBytes(viewerContent?.totalBytes ?? viewerFile?.uploadedBytes) }}
|
||||||
|
<template v-if="viewerContent"> · 当前展示 {{ formatBytes(viewerContent.shownBytes) }}</template>
|
||||||
|
<template v-if="viewerContent?.truncated"> · 更早内容未加载</template>
|
||||||
|
</span>
|
||||||
|
<span class="viewer-actions">
|
||||||
|
<button class="btn btn-sm btn-ghost" type="button" :disabled="viewerLoading" @click="loadContent(true)">刷新</button>
|
||||||
|
<button
|
||||||
|
class="btn btn-sm btn-ghost"
|
||||||
|
type="button"
|
||||||
|
:disabled="viewerLoading || !viewerContent?.truncated || viewerMaxBytes >= TAIL_MAX"
|
||||||
|
@click="loadMore"
|
||||||
|
>加载更早</button>
|
||||||
|
<a v-if="viewerFile" class="btn btn-sm btn-ghost dl-link" :href="deviceLogDownloadUrl(viewerFile.id)" download>下载完整日志</a>
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<pre ref="viewerPre" class="log-pre">{{ viewerLoading && !viewerContent ? '加载中...' : (viewerContent?.content || '(暂无内容)') }}</pre>
|
||||||
|
</el-dialog>
|
||||||
|
|
||||||
|
<el-dialog v-model="configVisible" title="采集配置" width="720px">
|
||||||
|
<div class="config-block">
|
||||||
|
<h4>全局默认模式</h4>
|
||||||
|
<p class="config-desc">对未单独配置的终端生效。全量=上传日志目录内全部文件;精选=排除低价值大日志(客户端排除 pywebview;麦象排除 kk-browser / 控制台 / 测试日志)。终端在下一次上报周期(约 1 分钟内)跟随新配置。</p>
|
||||||
|
<div class="mode-switch">
|
||||||
|
<button
|
||||||
|
v-for="option in MODE_OPTIONS"
|
||||||
|
:key="option.value"
|
||||||
|
class="mode-btn"
|
||||||
|
:class="{ 'is-active': globalMode === option.value }"
|
||||||
|
type="button"
|
||||||
|
@click="saveGlobalMode(option.value)"
|
||||||
|
>{{ option.label }}</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="config-block">
|
||||||
|
<h4>终端覆盖</h4>
|
||||||
|
<div class="override-add">
|
||||||
|
<select v-model="newOverrideKey" class="override-select">
|
||||||
|
<option value="">选择终端(最近上报的设备)</option>
|
||||||
|
<option v-for="device in devices" :key="`${device.source}|${device.deviceId}`" :value="`${device.source}|${device.deviceId}`">
|
||||||
|
{{ sourceLabel(device.source) }} · {{ device.deviceName || device.deviceId }}({{ device.deviceId }})
|
||||||
|
</option>
|
||||||
|
</select>
|
||||||
|
<select v-model="newOverrideMode">
|
||||||
|
<option v-for="option in MODE_OPTIONS" :key="option.value" :value="option.value">{{ option.label }}</option>
|
||||||
|
</select>
|
||||||
|
<button class="btn btn-sm" type="button" @click="addOverride">添加/更新覆盖</button>
|
||||||
|
</div>
|
||||||
|
<table class="override-table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th style="width: 100px">来源</th>
|
||||||
|
<th>设备</th>
|
||||||
|
<th style="width: 80px">模式</th>
|
||||||
|
<th style="width: 160px">更新时间</th>
|
||||||
|
<th style="width: 90px">操作</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr v-for="row in overrides" :key="row.id">
|
||||||
|
<td>{{ sourceLabel(row.source) }}</td>
|
||||||
|
<td>
|
||||||
|
<span class="device-name">{{ row.deviceName || row.deviceId }}</span>
|
||||||
|
<span class="device-id">{{ row.deviceId }}</span>
|
||||||
|
</td>
|
||||||
|
<td>{{ modeLabel(row.mode) }}</td>
|
||||||
|
<td>{{ row.updatedAt ? formatDateTime(row.updatedAt) : '—' }}</td>
|
||||||
|
<td class="ops-cell">
|
||||||
|
<button class="btn btn-sm btn-danger" type="button" @click="removeOverride(row)">删除</button>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr v-if="!overrides.length">
|
||||||
|
<td colspan="5" class="empty-tip">{{ configLoading ? '加载中...' : '暂无终端覆盖(全部跟随全局默认)' }}</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
<template #footer>
|
||||||
|
<el-button @click="configVisible = false">关闭</el-button>
|
||||||
|
</template>
|
||||||
|
</el-dialog>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
/* 沿用「记录与版本」系列的旧后台面板视觉语言。 */
|
||||||
|
.device-logs-view {
|
||||||
|
font-family: inherit;
|
||||||
|
color: #24384d;
|
||||||
|
}
|
||||||
|
.panel-box {
|
||||||
|
width: 100%;
|
||||||
|
min-width: 0;
|
||||||
|
padding: 20px 22px 24px;
|
||||||
|
border: 1px solid #d8e3ee;
|
||||||
|
border-radius: 14px;
|
||||||
|
background: linear-gradient(145deg, #ffffff, #f9fbfd);
|
||||||
|
box-shadow: 0 1px 2px rgba(39, 67, 94, 0.04), 0 14px 30px -24px rgba(39, 67, 94, 0.28);
|
||||||
|
}
|
||||||
|
h3 {
|
||||||
|
margin: 0;
|
||||||
|
font-size: 15px;
|
||||||
|
font-weight: 650;
|
||||||
|
color: #24384d;
|
||||||
|
letter-spacing: 0.2px;
|
||||||
|
}
|
||||||
|
.logs-head {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 12px;
|
||||||
|
margin-bottom: 16px;
|
||||||
|
}
|
||||||
|
.logs-head-tools {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 10px;
|
||||||
|
}
|
||||||
|
.retention-tip {
|
||||||
|
color: #8598ab;
|
||||||
|
font-size: 12.5px;
|
||||||
|
}
|
||||||
|
.logs-filter-row {
|
||||||
|
margin: 0 0 16px;
|
||||||
|
}
|
||||||
|
.form-row {
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
align-items: flex-end;
|
||||||
|
gap: 14px 18px;
|
||||||
|
}
|
||||||
|
.form-group {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 7px;
|
||||||
|
}
|
||||||
|
.form-group label {
|
||||||
|
color: #5b6f83;
|
||||||
|
font-size: 12.5px;
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
.form-group input,
|
||||||
|
.form-group select {
|
||||||
|
min-width: 0;
|
||||||
|
min-height: 42px;
|
||||||
|
padding: 8px 12px;
|
||||||
|
border: 1px solid #cbd9e6;
|
||||||
|
border-radius: 9px;
|
||||||
|
background: #f8fbfd;
|
||||||
|
color: #24384d;
|
||||||
|
font-size: 13.5px;
|
||||||
|
font-family: inherit;
|
||||||
|
color-scheme: light;
|
||||||
|
outline: none;
|
||||||
|
}
|
||||||
|
.form-group input:focus,
|
||||||
|
.form-group select:focus {
|
||||||
|
background: #ffffff;
|
||||||
|
border-color: #5f85ad;
|
||||||
|
box-shadow: 0 0 0 3px rgba(95, 133, 173, 0.16);
|
||||||
|
}
|
||||||
|
.filter-actions {
|
||||||
|
display: flex;
|
||||||
|
gap: 10px;
|
||||||
|
}
|
||||||
|
.btn {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
gap: 8px;
|
||||||
|
min-height: 42px;
|
||||||
|
padding: 9px 18px;
|
||||||
|
border: 1px solid #4f78a5;
|
||||||
|
border-radius: 9px;
|
||||||
|
background: linear-gradient(135deg, #5f85ad, #4f78a5);
|
||||||
|
color: #ffffff;
|
||||||
|
font-family: inherit;
|
||||||
|
font-size: 13.5px;
|
||||||
|
cursor: pointer;
|
||||||
|
box-shadow: 0 8px 18px -14px rgba(79, 120, 165, 0.72);
|
||||||
|
text-decoration: none;
|
||||||
|
}
|
||||||
|
.btn:hover:not(:disabled) {
|
||||||
|
background: linear-gradient(135deg, #7094ba, #5d83ac);
|
||||||
|
}
|
||||||
|
.btn:disabled {
|
||||||
|
cursor: not-allowed;
|
||||||
|
opacity: 0.55;
|
||||||
|
}
|
||||||
|
.btn-ghost {
|
||||||
|
background: #ffffff;
|
||||||
|
border-color: #c7d7e5;
|
||||||
|
color: #4f78a5;
|
||||||
|
box-shadow: none;
|
||||||
|
}
|
||||||
|
.btn-ghost:hover:not(:disabled) {
|
||||||
|
background: #edf5fb;
|
||||||
|
border-color: #95b1cb;
|
||||||
|
color: #2f5d8b;
|
||||||
|
}
|
||||||
|
.btn-danger {
|
||||||
|
background: linear-gradient(135deg, #c06d77, #b35f6a);
|
||||||
|
border-color: #b35f6a;
|
||||||
|
}
|
||||||
|
.btn-danger:hover:not(:disabled) {
|
||||||
|
background: linear-gradient(135deg, #cb7c84, #b96570);
|
||||||
|
}
|
||||||
|
.btn-sm {
|
||||||
|
min-height: 32px;
|
||||||
|
padding: 4px 12px;
|
||||||
|
font-size: 12.5px;
|
||||||
|
border-radius: 7px;
|
||||||
|
}
|
||||||
|
.logs-table-scroll {
|
||||||
|
width: 100%;
|
||||||
|
overflow-x: auto;
|
||||||
|
}
|
||||||
|
table {
|
||||||
|
width: 100%;
|
||||||
|
border-collapse: collapse;
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
th {
|
||||||
|
padding: 10px 12px;
|
||||||
|
text-align: left;
|
||||||
|
color: #5b6f83;
|
||||||
|
font-weight: 650;
|
||||||
|
font-size: 12.5px;
|
||||||
|
border-bottom: 1px solid #dbe6f0;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
td {
|
||||||
|
padding: 10px 12px;
|
||||||
|
border-bottom: 1px solid #eaf1f7;
|
||||||
|
vertical-align: top;
|
||||||
|
}
|
||||||
|
.source-pill {
|
||||||
|
display: inline-block;
|
||||||
|
padding: 2px 10px;
|
||||||
|
border-radius: 999px;
|
||||||
|
font-size: 12px;
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
.source-pill.is-client {
|
||||||
|
background: #e8f1fb;
|
||||||
|
color: #37618f;
|
||||||
|
}
|
||||||
|
.source-pill.is-maixiang {
|
||||||
|
background: #eef7ec;
|
||||||
|
color: #3f7a42;
|
||||||
|
}
|
||||||
|
.device-name {
|
||||||
|
display: block;
|
||||||
|
font-weight: 600;
|
||||||
|
word-break: break-all;
|
||||||
|
}
|
||||||
|
.device-id {
|
||||||
|
display: block;
|
||||||
|
color: #8598ab;
|
||||||
|
font-size: 11.5px;
|
||||||
|
word-break: break-all;
|
||||||
|
}
|
||||||
|
.user-name {
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
.file-name {
|
||||||
|
display: block;
|
||||||
|
word-break: break-all;
|
||||||
|
}
|
||||||
|
.dim {
|
||||||
|
color: #9db0c2;
|
||||||
|
}
|
||||||
|
.ops-cell {
|
||||||
|
display: flex;
|
||||||
|
gap: 8px;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
border-bottom: none;
|
||||||
|
}
|
||||||
|
.empty-tip {
|
||||||
|
padding: 26px 0;
|
||||||
|
text-align: center;
|
||||||
|
color: #8598ab;
|
||||||
|
}
|
||||||
|
.dl-link {
|
||||||
|
text-decoration: none;
|
||||||
|
}
|
||||||
|
.viewer-toolbar {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 12px;
|
||||||
|
margin-bottom: 10px;
|
||||||
|
}
|
||||||
|
.viewer-meta {
|
||||||
|
color: #5b6f83;
|
||||||
|
font-size: 12.5px;
|
||||||
|
}
|
||||||
|
.viewer-actions {
|
||||||
|
display: flex;
|
||||||
|
gap: 8px;
|
||||||
|
}
|
||||||
|
.log-pre {
|
||||||
|
max-height: 62vh;
|
||||||
|
margin: 0;
|
||||||
|
padding: 12px 14px;
|
||||||
|
overflow: auto;
|
||||||
|
border: 1px solid #d8e3ee;
|
||||||
|
border-radius: 10px;
|
||||||
|
background: #0f1c29;
|
||||||
|
color: #d7e4f1;
|
||||||
|
font-family: Consolas, 'Courier New', monospace;
|
||||||
|
font-size: 12.5px;
|
||||||
|
line-height: 1.55;
|
||||||
|
white-space: pre;
|
||||||
|
}
|
||||||
|
.config-block {
|
||||||
|
margin-bottom: 18px;
|
||||||
|
}
|
||||||
|
.config-block h4 {
|
||||||
|
margin: 0 0 6px;
|
||||||
|
font-size: 13.5px;
|
||||||
|
color: #24384d;
|
||||||
|
}
|
||||||
|
.config-desc {
|
||||||
|
margin: 0 0 10px;
|
||||||
|
color: #66798d;
|
||||||
|
font-size: 12.5px;
|
||||||
|
line-height: 1.6;
|
||||||
|
}
|
||||||
|
.mode-switch {
|
||||||
|
display: flex;
|
||||||
|
gap: 10px;
|
||||||
|
}
|
||||||
|
.mode-btn {
|
||||||
|
min-height: 38px;
|
||||||
|
padding: 8px 20px;
|
||||||
|
border: 1px solid #c7d7e5;
|
||||||
|
border-radius: 9px;
|
||||||
|
background: #ffffff;
|
||||||
|
color: #4f78a5;
|
||||||
|
font-size: 13.5px;
|
||||||
|
font-family: inherit;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
.mode-btn.is-active {
|
||||||
|
border-color: #4f78a5;
|
||||||
|
background: linear-gradient(135deg, #5f85ad, #4f78a5);
|
||||||
|
color: #ffffff;
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
.override-add {
|
||||||
|
display: flex;
|
||||||
|
gap: 10px;
|
||||||
|
margin-bottom: 12px;
|
||||||
|
}
|
||||||
|
.override-add select {
|
||||||
|
min-height: 38px;
|
||||||
|
padding: 6px 10px;
|
||||||
|
border: 1px solid #cbd9e6;
|
||||||
|
border-radius: 9px;
|
||||||
|
background: #f8fbfd;
|
||||||
|
color: #24384d;
|
||||||
|
font-size: 13px;
|
||||||
|
font-family: inherit;
|
||||||
|
color-scheme: light;
|
||||||
|
}
|
||||||
|
.override-select {
|
||||||
|
flex: 1;
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
.override-table td {
|
||||||
|
vertical-align: middle;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -2,7 +2,8 @@
|
|||||||
import { formatDateTime } from '@/utils/datetime'
|
import { formatDateTime } from '@/utils/datetime'
|
||||||
/** 教程管理页:工具台首页「立即下载教程」的包体管理(上传新包 / 列表 / 下载 / 删除)。
|
/** 教程管理页:工具台首页「立即下载教程」的包体管理(上传新包 / 列表 / 下载 / 删除)。
|
||||||
* 上传走浏览器直传 MinIO(presign → PUT 带进度 → confirm 落库),与软件版本管理页同一套链路;
|
* 上传走浏览器直传 MinIO(presign → PUT 带进度 → confirm 落库),与软件版本管理页同一套链路;
|
||||||
* 工具台始终下载"最新上传"的包(列表第一条即当前生效)。 */
|
* 上传时可填版本号(仅展示与排序用);工具台始终下载"最新上传"的包(不随列表排序变化)。
|
||||||
|
* 列表默认按上传时间降序,「版本号」「上传时间」表头可点击切换升降序。 */
|
||||||
import { computed, onMounted, ref, watch } from 'vue'
|
import { computed, onMounted, ref, watch } from 'vue'
|
||||||
import OldPagination from '@/components/OldPagination.vue'
|
import OldPagination from '@/components/OldPagination.vue'
|
||||||
import { ElMessage } from 'element-plus'
|
import { ElMessage } from 'element-plus'
|
||||||
@@ -22,12 +23,64 @@ const filteredItems = computed(() => {
|
|||||||
/** 全站分页统一:客户端分页(10/20/50/100)。 */
|
/** 全站分页统一:客户端分页(10/20/50/100)。 */
|
||||||
const page = ref(1)
|
const page = ref(1)
|
||||||
const pageSize = ref(10)
|
const pageSize = ref(10)
|
||||||
const pagedItems = computed(() => filteredItems.value.slice((page.value - 1) * pageSize.value, page.value * pageSize.value))
|
|
||||||
|
/** 排序:默认按上传时间降序(最新上传在最前,与后端返回顺序一致);点表头切换升降序。 */
|
||||||
|
type SortKey = 'version' | 'createdAt'
|
||||||
|
const sortKey = ref<SortKey>('createdAt')
|
||||||
|
const sortAsc = ref(false)
|
||||||
|
|
||||||
|
function toggleSort(key: SortKey) {
|
||||||
|
if (sortKey.value === key) {
|
||||||
|
sortAsc.value = !sortAsc.value
|
||||||
|
} else {
|
||||||
|
sortKey.value = key
|
||||||
|
sortAsc.value = false
|
||||||
|
}
|
||||||
|
page.value = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 排序标记:未激活 ▲▼,激活时只留方向箭头(比 ⇅/↕ 字形支持好,避免 Windows 缺字形显示方框)。 */
|
||||||
|
function sortMark(key: SortKey): string {
|
||||||
|
if (sortKey.value !== key) return '▲▼'
|
||||||
|
return sortAsc.value ? '▲' : '▼'
|
||||||
|
}
|
||||||
|
|
||||||
|
function compareText(a: string, b: string): number {
|
||||||
|
return a.localeCompare(b, 'zh-Hans-CN', { numeric: true, sensitivity: 'base' })
|
||||||
|
}
|
||||||
|
|
||||||
|
const sortedItems = computed(() => {
|
||||||
|
const rows = [...filteredItems.value]
|
||||||
|
rows.sort((a, b) => {
|
||||||
|
// 版本号为空的历史行始终排在末尾,避免切换排序时"无版本"占满首页。
|
||||||
|
if (sortKey.value === 'version') {
|
||||||
|
const av = a.version
|
||||||
|
const bv = b.version
|
||||||
|
if (!av || !bv) {
|
||||||
|
if (!av && !bv) return 0
|
||||||
|
return av ? -1 : 1
|
||||||
|
}
|
||||||
|
const diff = compareText(av, bv)
|
||||||
|
return sortAsc.value ? diff : -diff
|
||||||
|
}
|
||||||
|
const at = a.createdAt
|
||||||
|
const bt = b.createdAt
|
||||||
|
if (!at || !bt) {
|
||||||
|
if (!at && !bt) return 0
|
||||||
|
return at ? -1 : 1
|
||||||
|
}
|
||||||
|
const diff = compareText(at, bt)
|
||||||
|
return sortAsc.value ? diff : -diff
|
||||||
|
})
|
||||||
|
return rows
|
||||||
|
})
|
||||||
|
|
||||||
|
const pagedItems = computed(() => sortedItems.value.slice((page.value - 1) * pageSize.value, page.value * pageSize.value))
|
||||||
function changePage(p: number) { page.value = p }
|
function changePage(p: number) { page.value = p }
|
||||||
function changeSize(size: number) { pageSize.value = size; page.value = 1 }
|
function changeSize(size: number) { pageSize.value = size; page.value = 1 }
|
||||||
// 搜索导致数据收缩时回钳页码,避免停在空页。
|
// 搜索/排序导致数据收缩时回钳页码,避免停在空页。
|
||||||
watch(filteredItems, () => {
|
watch(sortedItems, () => {
|
||||||
page.value = Math.min(page.value, Math.max(1, Math.ceil(filteredItems.value.length / pageSize.value)))
|
page.value = Math.min(page.value, Math.max(1, Math.ceil(sortedItems.value.length / pageSize.value)))
|
||||||
})
|
})
|
||||||
|
|
||||||
/** 当前生效包 = 列表第一条(工具台下发的就是它)。 */
|
/** 当前生效包 = 列表第一条(工具台下发的就是它)。 */
|
||||||
@@ -80,6 +133,7 @@ async function removeOne(row: TutorialPackageItem) {
|
|||||||
const uploadVisible = ref(false)
|
const uploadVisible = ref(false)
|
||||||
const uploading = ref(false)
|
const uploading = ref(false)
|
||||||
const uploadPercent = ref(0)
|
const uploadPercent = ref(0)
|
||||||
|
const newVersion = ref('')
|
||||||
const pickedFile = ref<File | null>(null)
|
const pickedFile = ref<File | null>(null)
|
||||||
/** 弹窗内成功/失败文案留驻(对齐软件版本页交互)。 */
|
/** 弹窗内成功/失败文案留驻(对齐软件版本页交互)。 */
|
||||||
const uploadMsg = ref('')
|
const uploadMsg = ref('')
|
||||||
@@ -103,6 +157,7 @@ function onFileChange(file: File) {
|
|||||||
function openUpload() {
|
function openUpload() {
|
||||||
uploadMsg.value = ''
|
uploadMsg.value = ''
|
||||||
uploadMsgOk.value = false
|
uploadMsgOk.value = false
|
||||||
|
newVersion.value = ''
|
||||||
pickedFile.value = null
|
pickedFile.value = null
|
||||||
uploadVisible.value = true
|
uploadVisible.value = true
|
||||||
}
|
}
|
||||||
@@ -126,12 +181,13 @@ async function submitUpload() {
|
|||||||
uploading.value = true
|
uploading.value = true
|
||||||
try {
|
try {
|
||||||
// 浏览器直传 MinIO:presign → PUT(进度条)→ confirm 落库。
|
// 浏览器直传 MinIO:presign → PUT(进度条)→ confirm 落库。
|
||||||
await uploadTutorialPackage(pickedFile.value, pickedFile.value.name, (p) => {
|
await uploadTutorialPackage(pickedFile.value, pickedFile.value.name, newVersion.value.trim(), (p) => {
|
||||||
uploadPercent.value = p
|
uploadPercent.value = p
|
||||||
uploadMsg.value = p >= 100 ? '上传完成,正在登记教程包...' : `正在上传:${p}%`
|
uploadMsg.value = p >= 100 ? '上传完成,正在登记教程包...' : `正在上传:${p}%`
|
||||||
})
|
})
|
||||||
uploadMsg.value = '上传成功,工具台「立即下载教程」已切换为该包'
|
uploadMsg.value = '上传成功,工具台「立即下载教程」已切换为该包'
|
||||||
uploadMsgOk.value = true
|
uploadMsgOk.value = true
|
||||||
|
newVersion.value = ''
|
||||||
pickedFile.value = null
|
pickedFile.value = null
|
||||||
uploadPercent.value = 0
|
uploadPercent.value = 0
|
||||||
load()
|
load()
|
||||||
@@ -165,8 +221,13 @@ onMounted(load)
|
|||||||
<input type="checkbox" :checked="pagedItems.length > 0 && pagedItems.every((r) => selectedIds.has(r.id))" @change="toggleSelectAllPage" :disabled="!pagedItems.length" />
|
<input type="checkbox" :checked="pagedItems.length > 0 && pagedItems.every((r) => selectedIds.has(r.id))" @change="toggleSelectAllPage" :disabled="!pagedItems.length" />
|
||||||
</th>
|
</th>
|
||||||
<th style="width: 260px">文件名</th>
|
<th style="width: 260px">文件名</th>
|
||||||
|
<th style="width: 110px">
|
||||||
|
<button class="sort-th sort-version" type="button" @click="toggleSort('version')">版本号<span class="sort-mark">{{ sortMark('version') }}</span></button>
|
||||||
|
</th>
|
||||||
<th style="width: 110px">大小</th>
|
<th style="width: 110px">大小</th>
|
||||||
<th style="width: 150px">上传时间</th>
|
<th style="width: 150px">
|
||||||
|
<button class="sort-th sort-time" type="button" @click="toggleSort('createdAt')">上传时间<span class="sort-mark">{{ sortMark('createdAt') }}</span></button>
|
||||||
|
</th>
|
||||||
<th>下载链接</th>
|
<th>下载链接</th>
|
||||||
<th style="width: 170px">操作</th>
|
<th style="width: 170px">操作</th>
|
||||||
</tr>
|
</tr>
|
||||||
@@ -181,6 +242,10 @@ onMounted(load)
|
|||||||
<span class="file-name" :title="row.fileName">{{ row.fileName }}</span>
|
<span class="file-name" :title="row.fileName">{{ row.fileName }}</span>
|
||||||
<span v-if="row.id === activeId" class="tag-active">当前生效</span>
|
<span v-if="row.id === activeId" class="tag-active">当前生效</span>
|
||||||
</td>
|
</td>
|
||||||
|
<td>
|
||||||
|
<span v-if="row.version" class="version-cell" :title="row.version">{{ row.version }}</span>
|
||||||
|
<span v-else class="dim">—</span>
|
||||||
|
</td>
|
||||||
<td>{{ formatFileSize(row.fileSize) }}</td>
|
<td>{{ formatFileSize(row.fileSize) }}</td>
|
||||||
<td>{{ formatDateTime(row.createdAt) }}</td>
|
<td>{{ formatDateTime(row.createdAt) }}</td>
|
||||||
<td>
|
<td>
|
||||||
@@ -195,10 +260,10 @@ onMounted(load)
|
|||||||
</tr>
|
</tr>
|
||||||
</template>
|
</template>
|
||||||
<tr v-else-if="loading">
|
<tr v-else-if="loading">
|
||||||
<td colspan="6" class="empty-tip">加载中...</td>
|
<td colspan="7" class="empty-tip">加载中...</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr v-else>
|
<tr v-else>
|
||||||
<td colspan="6" class="empty-tip">{{ keyword ? '暂无匹配教程包' : '暂无教程包记录' }}</td>
|
<td colspan="7" class="empty-tip">{{ keyword ? '暂无匹配教程包' : '暂无教程包记录' }}</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
@@ -209,6 +274,10 @@ onMounted(load)
|
|||||||
<el-dialog v-model="uploadVisible" title="上传教程包" width="520px">
|
<el-dialog v-model="uploadVisible" title="上传教程包" width="520px">
|
||||||
<p class="upload-desc">上传教程 ZIP 包后,工具台首页「立即下载教程」将以下载该包为准(以最新上传的为主)。</p>
|
<p class="upload-desc">上传教程 ZIP 包后,工具台首页「立即下载教程」将以下载该包为准(以最新上传的为主)。</p>
|
||||||
<el-form label-width="110px">
|
<el-form label-width="110px">
|
||||||
|
<el-form-item label="版本号">
|
||||||
|
<el-input v-model="newVersion" placeholder="例如:v2026.09 或留空" maxlength="64" />
|
||||||
|
<p class="zip-hint">仅作展示与排序用,可留空;不影响工具台按最新上传下载</p>
|
||||||
|
</el-form-item>
|
||||||
<el-form-item label="ZIP 压缩包" required>
|
<el-form-item label="ZIP 压缩包" required>
|
||||||
<el-upload :auto-upload="false" :limit="1" accept=".zip" :on-change="(file: any) => onFileChange(file.raw)" :on-remove="() => (pickedFile = null)">
|
<el-upload :auto-upload="false" :limit="1" accept=".zip" :on-change="(file: any) => onFileChange(file.raw)" :on-remove="() => (pickedFile = null)">
|
||||||
<el-button>选择文件</el-button>
|
<el-button>选择文件</el-button>
|
||||||
@@ -368,6 +437,46 @@ h3 {
|
|||||||
text-overflow: ellipsis;
|
text-overflow: ellipsis;
|
||||||
vertical-align: middle;
|
vertical-align: middle;
|
||||||
}
|
}
|
||||||
|
/* 表头排序:按钮铺满单元格,仅 hover 时加深字色,保持旧版表头观感。 */
|
||||||
|
.sort-th {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 4px;
|
||||||
|
width: 100%;
|
||||||
|
padding: 0;
|
||||||
|
border: 0;
|
||||||
|
background: none;
|
||||||
|
color: inherit;
|
||||||
|
font-family: inherit;
|
||||||
|
font-size: inherit;
|
||||||
|
font-weight: inherit;
|
||||||
|
letter-spacing: inherit;
|
||||||
|
text-align: left;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
.sort-th:hover {
|
||||||
|
color: #2f5d8b;
|
||||||
|
}
|
||||||
|
.sort-mark {
|
||||||
|
color: #8293a5;
|
||||||
|
font-size: 12px;
|
||||||
|
}
|
||||||
|
.sort-th:hover .sort-mark {
|
||||||
|
color: #5f85ad;
|
||||||
|
}
|
||||||
|
.sort-version {
|
||||||
|
width: 110px;
|
||||||
|
}
|
||||||
|
.sort-time {
|
||||||
|
width: 150px;
|
||||||
|
}
|
||||||
|
.version-cell {
|
||||||
|
display: inline-block;
|
||||||
|
max-width: 100%;
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
vertical-align: middle;
|
||||||
|
}
|
||||||
.tag-active {
|
.tag-active {
|
||||||
display: inline-block;
|
display: inline-block;
|
||||||
margin-left: 6px;
|
margin-left: 6px;
|
||||||
|
|||||||
@@ -22,11 +22,12 @@ export interface TutorialUploadTarget {
|
|||||||
objectKey: string
|
objectKey: string
|
||||||
uploadUrl: string
|
uploadUrl: string
|
||||||
fileUrl: string
|
fileUrl: string
|
||||||
|
version: string
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 第一步:向后端申请直传 PUT 预签名地址(登录会话签发,3 分钟有效)。 */
|
/** 第一步:向后端申请直传 PUT 预签名地址(登录会话签发,3 分钟有效)。 */
|
||||||
export async function requestTutorialPresign(fileName: string): Promise<TutorialUploadTarget> {
|
export async function requestTutorialPresign(fileName: string, version = ''): Promise<TutorialUploadTarget> {
|
||||||
const { data } = await http.post<unknown>(TUTORIAL_PRESIGN_ENDPOINT, null, { params: { file_name: fileName } })
|
const { data } = await http.post<unknown>(TUTORIAL_PRESIGN_ENDPOINT, null, { params: { file_name: fileName, version } })
|
||||||
const core = (unwrap(data) ?? {}) as Record<string, unknown>
|
const core = (unwrap(data) ?? {}) as Record<string, unknown>
|
||||||
const uploadUrl = typeof core.upload_url === 'string' ? core.upload_url : ''
|
const uploadUrl = typeof core.upload_url === 'string' ? core.upload_url : ''
|
||||||
const objectKey = typeof core.object_key === 'string' ? core.object_key : ''
|
const objectKey = typeof core.object_key === 'string' ? core.object_key : ''
|
||||||
@@ -37,13 +38,14 @@ export async function requestTutorialPresign(fileName: string): Promise<Tutorial
|
|||||||
objectKey,
|
objectKey,
|
||||||
uploadUrl,
|
uploadUrl,
|
||||||
fileUrl: typeof core.file_url === 'string' ? core.file_url : '',
|
fileUrl: typeof core.file_url === 'string' ? core.file_url : '',
|
||||||
|
version: typeof core.version === 'string' ? core.version : version.trim(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 第三步:直传完成后通知后端校验对象并写入记录(data.item 为新记录行)。 */
|
/** 第三步:直传完成后通知后端校验对象并写入记录(data.item 为新记录行)。 */
|
||||||
export async function confirmTutorialPackage(objectKey: string, fileName: string): Promise<TutorialPackageItem | null> {
|
export async function confirmTutorialPackage(objectKey: string, fileName: string, version = ''): Promise<TutorialPackageItem | null> {
|
||||||
const { data } = await http.post<unknown>(TUTORIAL_CONFIRM_ENDPOINT, null, {
|
const { data } = await http.post<unknown>(TUTORIAL_CONFIRM_ENDPOINT, null, {
|
||||||
params: { object_key: objectKey, file_name: fileName },
|
params: { object_key: objectKey, file_name: fileName, version },
|
||||||
})
|
})
|
||||||
return parseTutorialPackageUpload(data)
|
return parseTutorialPackageUpload(data)
|
||||||
}
|
}
|
||||||
@@ -63,9 +65,10 @@ export async function deleteTutorialPackages(ids: number[]): Promise<number> {
|
|||||||
export async function uploadTutorialPackage(
|
export async function uploadTutorialPackage(
|
||||||
file: Blob,
|
file: Blob,
|
||||||
fileName: string,
|
fileName: string,
|
||||||
|
version = '',
|
||||||
onProgress?: (percent: number) => void,
|
onProgress?: (percent: number) => void,
|
||||||
): Promise<TutorialPackageItem | null> {
|
): Promise<TutorialPackageItem | null> {
|
||||||
const target = await requestTutorialPresign(fileName)
|
const target = await requestTutorialPresign(fileName, version)
|
||||||
await directPut.put(target.uploadUrl, file, {
|
await directPut.put(target.uploadUrl, file, {
|
||||||
headers: { 'Content-Type': 'application/octet-stream' },
|
headers: { 'Content-Type': 'application/octet-stream' },
|
||||||
onUploadProgress: (event) => {
|
onUploadProgress: (event) => {
|
||||||
@@ -74,5 +77,5 @@ export async function uploadTutorialPackage(
|
|||||||
onProgress(total > 0 ? Math.min(Math.round((event.loaded / total) * 100), 100) : 0)
|
onProgress(total > 0 ? Math.min(Math.round((event.loaded / total) * 100), 100) : 0)
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
return confirmTutorialPackage(target.objectKey, fileName)
|
return confirmTutorialPackage(target.objectKey, fileName, target.version)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
export interface TutorialPackageItem {
|
export interface TutorialPackageItem {
|
||||||
id: number
|
id: number
|
||||||
fileName: string
|
fileName: string
|
||||||
|
/** 版本号(上传时填写,V126 之前的历史行为空串) */
|
||||||
|
version: string
|
||||||
objectKey: string
|
objectKey: string
|
||||||
fileSize: number
|
fileSize: number
|
||||||
fileUrl: string
|
fileUrl: string
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ export function toTutorialPackageItem(raw: unknown): TutorialPackageItem | null
|
|||||||
return {
|
return {
|
||||||
id,
|
id,
|
||||||
fileName: text(r.file_name ?? r.fileName),
|
fileName: text(r.file_name ?? r.fileName),
|
||||||
|
version: text(r.version),
|
||||||
objectKey: text(r.object_key ?? r.objectKey),
|
objectKey: text(r.object_key ?? r.objectKey),
|
||||||
fileSize: numberOrNull(r.file_size ?? r.fileSize) ?? 0,
|
fileSize: numberOrNull(r.file_size ?? r.fileSize) ?? 0,
|
||||||
fileUrl: text(r.file_url ?? r.fileUrl),
|
fileUrl: text(r.file_url ?? r.fileUrl),
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ export const adminPages: AdminPageDef[] = [
|
|||||||
{ path: 'records/history', menuKey: 'admin_history', title: '生成记录', load: () => import('@/pages/records/RecordsHistoryPage.vue') },
|
{ path: 'records/history', menuKey: 'admin_history', title: '生成记录', load: () => import('@/pages/records/RecordsHistoryPage.vue') },
|
||||||
{ path: 'records/software-version', menuKey: 'admin_version', title: '软件版本管理', load: () => import('@/pages/records/RecordsSoftwareVersionPage.vue') },
|
{ path: 'records/software-version', menuKey: 'admin_version', title: '软件版本管理', load: () => import('@/pages/records/RecordsSoftwareVersionPage.vue') },
|
||||||
{ path: 'records/tutorial', menuKey: 'admin_tutorial', title: '教程管理', load: () => import('@/pages/records/RecordsTutorialPage.vue') },
|
{ path: 'records/tutorial', menuKey: 'admin_tutorial', title: '教程管理', load: () => import('@/pages/records/RecordsTutorialPage.vue') },
|
||||||
|
{ path: 'records/device-logs', menuKey: 'admin_device_logs', title: '日志管理', load: () => import('@/pages/records/DeviceLogsPage.vue') },
|
||||||
{ path: 'records/digital-human-version', menuKey: 'digital_human_version', title: '数字人版本管理', load: () => import('@/pages/records/RecordsDigitalHumanVersionPage.vue') },
|
{ path: 'records/digital-human-version', menuKey: 'digital_human_version', title: '数字人版本管理', load: () => import('@/pages/records/RecordsDigitalHumanVersionPage.vue') },
|
||||||
{ path: 'records/image-video-tasks', menuKey: 'admin_image_video_tasks', title: '视频任务记录', load: () => import('@/pages/tasks/ImageVideoTasksPage.vue') },
|
{ path: 'records/image-video-tasks', menuKey: 'admin_image_video_tasks', title: '视频任务记录', load: () => import('@/pages/tasks/ImageVideoTasksPage.vue') },
|
||||||
{ path: 'shop-center/data-tasks', menuKey: 'admin_shop_data_crawl_tasks', title: '店铺数据记录', load: () => import('@/pages/tasks/ShopDataTasksPage.vue') },
|
{ path: 'shop-center/data-tasks', menuKey: 'admin_shop_data_crawl_tasks', title: '店铺数据记录', load: () => import('@/pages/tasks/ShopDataTasksPage.vue') },
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ export interface AdminUser {
|
|||||||
createdById?: number | null
|
createdById?: number | null
|
||||||
creatorUsername?: string
|
creatorUsername?: string
|
||||||
createdAt?: string
|
createdAt?: string
|
||||||
|
updatedAt?: string
|
||||||
pinyinAbbr?: string
|
pinyinAbbr?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -82,7 +82,9 @@ test('align_query_asin_page_layout_wiring', () => {
|
|||||||
assert.match(page, /新增 ASIN/, '顶栏补新增 ASIN 按钮')
|
assert.match(page, /新增 ASIN/, '顶栏补新增 ASIN 按钮')
|
||||||
assert.match(page, /:rowspan="row\.rowspan"/, '表格用原生 rowspan 合并(像素复刻旧版)')
|
assert.match(page, /:rowspan="row\.rowspan"/, '表格用原生 rowspan 合并(像素复刻旧版)')
|
||||||
assert.match(page, /CopyText/, 'ASIN 单元格点击复制')
|
assert.match(page, /CopyText/, 'ASIN 单元格点击复制')
|
||||||
assert.doesNotMatch(page, /更新时间/, '去掉更新时间列(参考无此列)')
|
// 需求变更:后台管理列表统一展示「创建时间 + 更新时间」,覆盖早期「不加更新时间」的像素对齐。
|
||||||
|
assert.match(page, /创建时间/, '表格含创建时间列')
|
||||||
|
assert.match(page, /更新时间/, '表格含更新时间列')
|
||||||
assert.match(page, /queryAsinDisplayRows/, '行展开走纯模型')
|
assert.match(page, /queryAsinDisplayRows/, '行展开走纯模型')
|
||||||
assert.match(page, /请完整填写分组、店铺名、国家和 ASIN/, '新增弹窗校验文案对齐')
|
assert.match(page, /请完整填写分组、店铺名、国家和 ASIN/, '新增弹窗校验文案对齐')
|
||||||
assert.match(page, /请先选择分组/, '店铺下拉未选分组时占位对齐')
|
assert.match(page, /请先选择分组/, '店铺下拉未选分组时占位对齐')
|
||||||
|
|||||||
@@ -82,7 +82,9 @@ test('align_skip_price_page_layout_wiring', () => {
|
|||||||
assert.match(page, /新增 ASIN/, '顶栏补新增 ASIN 按钮')
|
assert.match(page, /新增 ASIN/, '顶栏补新增 ASIN 按钮')
|
||||||
assert.match(page, /:rowspan="row\.rowspan"/, '表格用原生 rowspan 合并(像素复刻旧版)')
|
assert.match(page, /:rowspan="row\.rowspan"/, '表格用原生 rowspan 合并(像素复刻旧版)')
|
||||||
assert.match(page, /CopyText/, 'ASIN 单元格点击复制')
|
assert.match(page, /CopyText/, 'ASIN 单元格点击复制')
|
||||||
assert.doesNotMatch(page, /更新时间/, '去掉更新时间列(参考无此列)')
|
// 需求变更:后台管理列表统一展示「创建时间 + 更新时间」,覆盖早期「不加更新时间」的像素对齐。
|
||||||
|
assert.match(page, /创建时间/, '表格含创建时间列')
|
||||||
|
assert.match(page, /更新时间/, '表格含更新时间列')
|
||||||
assert.match(page, /skipPriceDisplayRows/, '行展开走纯模型')
|
assert.match(page, /skipPriceDisplayRows/, '行展开走纯模型')
|
||||||
assert.match(page, /请完整填写分组、店铺名、国家和 ASIN/, '新增弹窗校验文案对齐')
|
assert.match(page, /请完整填写分组、店铺名、国家和 ASIN/, '新增弹窗校验文案对齐')
|
||||||
assert.match(page, /最低价格式不正确/, '最低价格式校验对齐')
|
assert.match(page, /最低价格式不正确/, '最低价格式校验对齐')
|
||||||
|
|||||||
@@ -14,16 +14,23 @@ test('align_tutorial_page_registered', () => {
|
|||||||
|
|
||||||
test('align_tutorial_page_wiring', () => {
|
test('align_tutorial_page_wiring', () => {
|
||||||
const page = readSource('src/pages/records/RecordsTutorialPage.vue')
|
const page = readSource('src/pages/records/RecordsTutorialPage.vue')
|
||||||
// 上传入口:选择 zip → 直传(进度条)→ 成功提示留驻弹窗。
|
// 上传入口:版本号(可空)→ 选择 zip → 直传(进度条)→ 成功提示留驻弹窗。
|
||||||
assert.match(page, /上传教程包/, '存在上传入口按钮')
|
assert.match(page, /上传教程包/, '存在上传入口按钮')
|
||||||
assert.match(page, /uploadTutorialPackage/, '上传走教程包直传链路')
|
assert.match(page, /uploadTutorialPackage/, '上传走教程包直传链路')
|
||||||
|
assert.match(page, /label="版本号"/, '上传弹窗提供版本号输入')
|
||||||
|
assert.match(page, /newVersion\.value\.trim\(\)/, '版本号去空白后随上传提交')
|
||||||
assert.match(page, /请选择 zip 压缩包/, '未选文件时提示')
|
assert.match(page, /请选择 zip 压缩包/, '未选文件时提示')
|
||||||
assert.match(page, /仅支持 \.zip 格式/, '格式校验提示')
|
assert.match(page, /仅支持 \.zip 格式/, '格式校验提示')
|
||||||
assert.match(page, /上传成功,工具台「立即下载教程」已切换为该包/, '成功提示说明生效位置')
|
assert.match(page, /上传成功,工具台「立即下载教程」已切换为该包/, '成功提示说明生效位置')
|
||||||
assert.match(page, /上传完成,正在登记教程包/, '进度文案')
|
assert.match(page, /上传完成,正在登记教程包/, '进度文案')
|
||||||
// 列表:当前生效标记 + 下载 + 删除 + 空态。
|
// 列表:版本号列 + 上传时间列 + 当前生效标记 + 下载 + 删除 + 空态。
|
||||||
assert.match(page, /当前生效/, '最新上传的包标记当前生效')
|
assert.match(page, /当前生效/, '最新上传的包标记当前生效')
|
||||||
assert.match(page, /formatFileSize/, '展示包体大小')
|
assert.match(page, /formatFileSize/, '展示包体大小')
|
||||||
|
assert.match(page, /row\.version/, '展示版本号列')
|
||||||
|
assert.match(page, /toggleSort\('version'\)/, '版本号表头可点击排序')
|
||||||
|
assert.match(page, /toggleSort\('createdAt'\)/, '上传时间表头可点击排序')
|
||||||
|
assert.match(page, /const sortKey = ref<SortKey>\('createdAt'\)/, '默认排序字段为上传时间')
|
||||||
|
assert.match(page, /const sortAsc = ref\(false\)/, '默认降序')
|
||||||
assert.match(page, /下载/, '操作列提供下载')
|
assert.match(page, /下载/, '操作列提供下载')
|
||||||
assert.match(page, /<a v-if="row\.fileUrl" class="btn btn-sm dl-btn"[^>]*download/, '下载链接带 download 强制下载')
|
assert.match(page, /<a v-if="row\.fileUrl" class="btn btn-sm dl-btn"[^>]*download/, '下载链接带 download 强制下载')
|
||||||
assert.match(page, /确认删除选中的/, '批量删除二次确认')
|
assert.match(page, /确认删除选中的/, '批量删除二次确认')
|
||||||
@@ -37,7 +44,8 @@ test('align_tutorial_api_contract', () => {
|
|||||||
assert.match(api, /\$\{TUTORIAL_UPLOAD_ENDPOINT\}\/presign/, '直传预签名端点')
|
assert.match(api, /\$\{TUTORIAL_UPLOAD_ENDPOINT\}\/presign/, '直传预签名端点')
|
||||||
assert.match(api, /\$\{TUTORIAL_UPLOAD_ENDPOINT\}\/confirm/, '确认端点')
|
assert.match(api, /\$\{TUTORIAL_UPLOAD_ENDPOINT\}\/confirm/, '确认端点')
|
||||||
assert.match(api, /\$\{TUTORIAL_UPLOAD_ENDPOINT\}\/delete/, '删除端点')
|
assert.match(api, /\$\{TUTORIAL_UPLOAD_ENDPOINT\}\/delete/, '删除端点')
|
||||||
assert.match(api, /params: \{ object_key: objectKey, file_name: fileName \}/, '确认回传对象 key 与文件名')
|
assert.match(api, /params: \{ file_name: fileName, version \}/, '预签名回传文件名与版本号')
|
||||||
|
assert.match(api, /params: \{ object_key: objectKey, file_name: fileName, version \}/, '确认回传对象 key、文件名与版本号')
|
||||||
assert.match(api, /withCredentials: false/, '直传实例不挂会话拦截器')
|
assert.match(api, /withCredentials: false/, '直传实例不挂会话拦截器')
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -46,4 +54,5 @@ test('align_tutorial_model_parsers', () => {
|
|||||||
assert.match(model, /r\.file_name \?\? r\.fileName/, '兼容 snake/camel 文件名')
|
assert.match(model, /r\.file_name \?\? r\.fileName/, '兼容 snake/camel 文件名')
|
||||||
assert.match(model, /r\.file_url \?\? r\.fileUrl/, '兼容 snake/camel 下载链接')
|
assert.match(model, /r\.file_url \?\? r\.fileUrl/, '兼容 snake/camel 下载链接')
|
||||||
assert.match(model, /r\.file_size \?\? r\.fileSize/, '兼容 snake/camel 文件大小')
|
assert.match(model, /r\.file_size \?\? r\.fileSize/, '兼容 snake/camel 文件大小')
|
||||||
|
assert.match(model, /version: text\(r\.version\)/, '解析版本号(缺失为空串,兼容历史行)')
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -0,0 +1,86 @@
|
|||||||
|
import test from 'node:test'
|
||||||
|
import assert from 'node:assert/strict'
|
||||||
|
import { readSource } from './helpers.ts'
|
||||||
|
import {
|
||||||
|
buildMenuOptionTree,
|
||||||
|
compactDirectGrantIds,
|
||||||
|
parseMenuOptionList,
|
||||||
|
parsePermissionMenuItem,
|
||||||
|
} from '../src/pages/account/user-menu-auth.ts'
|
||||||
|
|
||||||
|
// 2026-09-16 线上事故:非超管(普通管理员)在授权树里勾到自己无权授予的菜单后,
|
||||||
|
// 后端 ensureGrantable 抛 403 并回滚整笔事务——创建用户与保存权限双双失败,
|
||||||
|
// 前端只显示「普通管理员只能分配自己已有的菜单权限」。
|
||||||
|
// 修复:菜单列表按操作者标记 grantable,前端把不可授予的节点置灰不可勾。
|
||||||
|
//
|
||||||
|
// 注意:这里是「置灰」而非「隐藏」。授权保存是整树替换,隐藏会让超管早先授予、
|
||||||
|
// 而操作者自己没有的菜单在提交时被当作取消勾选删掉(与 09-13「权限自己没掉」同类)。
|
||||||
|
|
||||||
|
test('align_user_menu_grantable_false_maps_to_disabled_node', () => {
|
||||||
|
const locked = parsePermissionMenuItem(
|
||||||
|
{ id: 5, name: '查询ASIN', parent_id: null, sort_order: 1, grantable: false },
|
||||||
|
'admin',
|
||||||
|
)
|
||||||
|
assert.equal(locked?.disabled, true, 'grantable=false → 节点置灰')
|
||||||
|
|
||||||
|
const allowed = parsePermissionMenuItem(
|
||||||
|
{ id: 6, name: '店铺管理', parent_id: null, sort_order: 2, grantable: true },
|
||||||
|
'admin',
|
||||||
|
)
|
||||||
|
assert.equal(allowed?.disabled, false, 'grantable=true → 可勾选')
|
||||||
|
|
||||||
|
// 菜单管理页等未标记 grantable 的接口必须保持旧行为(全部可勾选)
|
||||||
|
const unmarked = parsePermissionMenuItem({ id: 7, name: '菜单权限配置', parent_id: null, sort_order: 3 }, 'admin')
|
||||||
|
assert.equal(unmarked?.disabled, false, '缺省 grantable 视为可授予')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('align_user_menu_grantable_survives_tree_build', () => {
|
||||||
|
const nodes = parseMenuOptionList(
|
||||||
|
[
|
||||||
|
{ id: 98, name: '账号与权限', parent_id: null, sort_order: 1, grantable: false },
|
||||||
|
{ id: 7, name: '用户管理', parent_id: 98, sort_order: 1, grantable: false },
|
||||||
|
{ id: 100, name: '店铺管理', parent_id: null, sort_order: 2, grantable: true },
|
||||||
|
],
|
||||||
|
'admin',
|
||||||
|
)
|
||||||
|
const tree = buildMenuOptionTree(nodes)
|
||||||
|
const account = tree.find((node) => node.id === 98)
|
||||||
|
assert.equal(account?.disabled, true, '分组节点置灰')
|
||||||
|
assert.equal(account?.children?.[0]?.disabled, true, '子节点置灰随树保留')
|
||||||
|
assert.equal(tree.find((node) => node.id === 100)?.disabled, false, '可授予节点不受影响')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('align_user_menu_grantable_disabled_node_still_compactable', () => {
|
||||||
|
// 已持有但无权授予的节点会保持勾选并原样提交,压缩逻辑不能因 disabled 漏掉它
|
||||||
|
const tree = buildMenuOptionTree(
|
||||||
|
parseMenuOptionList(
|
||||||
|
[
|
||||||
|
{ id: 98, name: '账号与权限', parent_id: null, sort_order: 1, grantable: false },
|
||||||
|
{ id: 7, name: '用户管理', parent_id: 98, sort_order: 1, grantable: false },
|
||||||
|
],
|
||||||
|
'admin',
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert.deepEqual(compactDirectGrantIds([98, 7], tree), [98], '父级已勾选时仍压缩掉后代')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('align_user_menu_grantable_wired_end_to_end', () => {
|
||||||
|
const tree = readSource('src/pages/account/UserMenuAuthTree.vue')
|
||||||
|
assert.match(tree, /disabled: 'disabled'/, 'el-tree 按 disabled 键置灰节点')
|
||||||
|
|
||||||
|
const vo = readSource(
|
||||||
|
'../backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/vo/PermissionMenuItemVo.java',
|
||||||
|
)
|
||||||
|
assert.match(vo, /private Boolean grantable;/, 'VO 暴露 grantable')
|
||||||
|
|
||||||
|
const controller = readSource(
|
||||||
|
'../backend-java/src/main/java/com/nanri/aiimage/modules/permission/controller/PermissionMenuController.java',
|
||||||
|
)
|
||||||
|
assert.match(controller, /permissionMenuService\.list\(requireAdmin\(request\), menuType\)/, '列表接口传入操作者')
|
||||||
|
|
||||||
|
const service = readSource(
|
||||||
|
'../backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java',
|
||||||
|
)
|
||||||
|
assert.match(service, /resolveGrantableMenuIds/, '按操作者计算可授予集')
|
||||||
|
assert.match(service, /ensureGrantable\(operator, grantIds, userId\)/, '保存校验传入目标用户以放行既有授权')
|
||||||
|
})
|
||||||
@@ -25,4 +25,12 @@ public final class BusinessCodes {
|
|||||||
|
|
||||||
/** 任务归属其它实例,需转发。 */
|
/** 任务归属其它实例,需转发。 */
|
||||||
public static final int TASK_OWNER_FORWARD = 40903;
|
public static final int TASK_OWNER_FORWARD = 40903;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 提交结果的目标任务已不存在(通常是被删除)。
|
||||||
|
* 语义:本次提交无意义,响应 success=false 且带该码,调用方应放弃而不是反复重试。
|
||||||
|
* 与 {@link #TASK_ALREADY_FINISHED} 的区别:那个还留了任务记录(可幂等忽略),
|
||||||
|
* 这个任务已经没了——如实报错,否则任务被误删时结果会被静默吞掉。
|
||||||
|
*/
|
||||||
|
public static final int TASK_NOT_FOUND = 40401;
|
||||||
}
|
}
|
||||||
|
|||||||
+40
-5
@@ -1,18 +1,21 @@
|
|||||||
package com.nanri.aiimage.common.exception;
|
package com.nanri.aiimage.common.exception;
|
||||||
|
|
||||||
import com.nanri.aiimage.common.api.ApiResponse;
|
import com.nanri.aiimage.common.api.ApiResponse;
|
||||||
|
import com.nanri.aiimage.common.security.DeviceSessionPolicy;
|
||||||
import com.nanri.aiimage.common.service.TaskOwnerForwardService;
|
import com.nanri.aiimage.common.service.TaskOwnerForwardService;
|
||||||
import com.nanri.aiimage.config.TaskOperationLockConfig;
|
import com.nanri.aiimage.config.TaskOperationLockConfig;
|
||||||
import jakarta.servlet.http.HttpServletRequest;
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
import jakarta.validation.ConstraintViolationException;
|
import jakarta.validation.ConstraintViolationException;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.http.HttpStatus;
|
||||||
import org.springframework.http.ResponseEntity;
|
import org.springframework.http.ResponseEntity;
|
||||||
import org.springframework.web.bind.MethodArgumentNotValidException;
|
import org.springframework.web.bind.MethodArgumentNotValidException;
|
||||||
import org.springframework.web.bind.annotation.ExceptionHandler;
|
import org.springframework.web.bind.annotation.ExceptionHandler;
|
||||||
import org.springframework.web.bind.annotation.RestControllerAdvice;
|
import org.springframework.web.bind.annotation.RestControllerAdvice;
|
||||||
import org.springframework.web.context.request.async.AsyncRequestNotUsableException;
|
import org.springframework.web.context.request.async.AsyncRequestNotUsableException;
|
||||||
import org.springframework.web.server.ResponseStatusException;
|
import org.springframework.web.server.ResponseStatusException;
|
||||||
|
import org.springframework.web.servlet.resource.NoResourceFoundException;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
@@ -52,22 +55,40 @@ public class GlobalExceptionHandler {
|
|||||||
? ApiResponse.fail(forwardEx.getMessage())
|
? ApiResponse.fail(forwardEx.getMessage())
|
||||||
: ApiResponse.fail(forwardEx.getCode(), forwardEx.getMessage());
|
: ApiResponse.fail(forwardEx.getCode(), forwardEx.getMessage());
|
||||||
} catch (Exception forwardEx) {
|
} catch (Exception forwardEx) {
|
||||||
|
// 转发失败是**瞬时基础设施故障**(归属实例正在滚动重启),不是业务结论,
|
||||||
|
// 更不能表达成「任务不存活」。原先返回 ApiResponse.fail(40903) —— HTTP 200
|
||||||
|
// 加 data:null,而客户端那句 bool((resp.json().get("data") or {}).get("alive"))
|
||||||
|
// 会把「拿不到数据」折叠成 alive=false,于是客户端把**健康的长任务主动停掉**:
|
||||||
|
// 2026-09-18 任务 28616 就是这么死的(归属节点 server-110 重启窗口内,心跳经
|
||||||
|
// nginx 落到 server-121,转发 3 次 Connection refused 后返回空 data)。
|
||||||
|
// 改为 503 + 空 body:新客户端按状态码判为「未知」继续跑;老客户端因 body 不是
|
||||||
|
// JSON、resp.json() 抛异常,同样落到「未知」。顺带让这类故障在 HTTP 指标里可见
|
||||||
|
// (原先记成 200,监控完全看不到滚动重启期间丢了多少心跳)。
|
||||||
log.warn("[instance-routing] forward failed taskId={} operation={} owner={} current={} msg={}",
|
log.warn("[instance-routing] forward failed taskId={} operation={} owner={} current={} msg={}",
|
||||||
ex.getTaskId(), ex.getOperation(), ex.getOwnerInstanceId(), ex.getCurrentInstanceId(),
|
ex.getTaskId(), ex.getOperation(), ex.getOwnerInstanceId(), ex.getCurrentInstanceId(),
|
||||||
forwardEx.getMessage(), forwardEx);
|
forwardEx.getMessage(), forwardEx);
|
||||||
return ApiResponse.fail(40903, "任务归属实例转发失败: " + forwardEx.getMessage());
|
return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).build();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ExceptionHandler(BusinessException.class)
|
@ExceptionHandler(BusinessException.class)
|
||||||
public ApiResponse<Void> handleBusinessException(BusinessException ex) {
|
public ApiResponse<Void> handleBusinessException(BusinessException ex, HttpServletRequest request) {
|
||||||
|
// 业务异常此前完全不记日志:2026-09-16 线上「保存权限/创建用户」双双失败时,
|
||||||
|
// 服务端只留 RequestTraceFilter 的 200 一行,根因只能靠反推响应体字节数才定位到。
|
||||||
|
// 401/4011(未登录、被顶下线)属于轮询类接口的常态噪声,降为 debug 以免淹没真实业务错。
|
||||||
|
if (isRoutineAuthNoise(ex.getCode())) {
|
||||||
|
log.debug("[business] {} {} code={} message={}", request.getMethod(), request.getRequestURI(),
|
||||||
|
ex.getCode(), ex.getMessage());
|
||||||
|
} else {
|
||||||
|
log.warn("[business] {} {} code={} message={}", request.getMethod(), request.getRequestURI(),
|
||||||
|
ex.getCode(), ex.getMessage());
|
||||||
|
}
|
||||||
if (Integer.valueOf(BusinessCodes.TASK_ALREADY_FINISHED).equals(ex.getCode())) {
|
if (Integer.valueOf(BusinessCodes.TASK_ALREADY_FINISHED).equals(ex.getCode())) {
|
||||||
// 幂等忽略:任务已结束时的重复提交无副作用,按成功返回,避免客户端反复重试
|
// 幂等忽略:任务已结束时的重复提交无副作用,按成功返回,避免客户端反复重试
|
||||||
return ApiResponse.success("任务已结束,忽略重复提交", null);
|
return ApiResponse.success("任务已结束,忽略重复提交", null);
|
||||||
}
|
}
|
||||||
if (Integer.valueOf(BusinessCodes.TASK_BUSY).equals(ex.getCode())) {
|
if (Integer.valueOf(BusinessCodes.TASK_BUSY).equals(ex.getCode())) {
|
||||||
// 锁竞争:必须如实返回失败 + 可重试码,否则 worker 会把「未落库」当成功而停止重试
|
// 锁竞争:必须如实返回失败 + 可重试码,否则 worker 会把「未落库」当成功而停止重试
|
||||||
log.warn("[business] 任务忙,调用方应稍后重试: {}", ex.getMessage());
|
|
||||||
return ApiResponse.fail(BusinessCodes.TASK_BUSY, ex.getMessage());
|
return ApiResponse.fail(BusinessCodes.TASK_BUSY, ex.getMessage());
|
||||||
}
|
}
|
||||||
return ex.getCode() == null
|
return ex.getCode() == null
|
||||||
@@ -75,9 +96,23 @@ public class GlobalExceptionHandler {
|
|||||||
: ApiResponse.fail(ex.getCode(), ex.getMessage());
|
: ApiResponse.fail(ex.getCode(), ex.getMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ExceptionHandler(NoResourceFoundException.class)
|
||||||
|
public ResponseEntity<ApiResponse<Void>> handleNoResourceFoundException(NoResourceFoundException ex) {
|
||||||
|
// 静态资源 404。绝大部分是外部扫描器在探测 /.env、/credentials、aliyun.json、oss.json
|
||||||
|
// 这类云凭据文件(线上单节点一天 580 条)。此前落到 handleException 里,既刷 ERROR 堆栈,
|
||||||
|
// 又把探测响应伪装成 HTTP 200;这里降为 debug 并如实返回 404。
|
||||||
|
log.debug("static resource not found: {}", ex.getMessage());
|
||||||
|
return ResponseEntity.status(HttpStatus.NOT_FOUND).body(ApiResponse.fail("资源不存在"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 未登录 / 登录态失效 / 被其他设备顶下线:按 401 语义的常态噪声,不占 WARN。 */
|
||||||
|
private boolean isRoutineAuthNoise(Integer code) {
|
||||||
|
return Integer.valueOf(401).equals(code)
|
||||||
|
|| Integer.valueOf(DeviceSessionPolicy.CODE_KICKED).equals(code);
|
||||||
|
}
|
||||||
|
|
||||||
@ExceptionHandler(MethodArgumentNotValidException.class)
|
@ExceptionHandler(MethodArgumentNotValidException.class)
|
||||||
public ApiResponse<Void> handleMethodArgumentNotValidException(MethodArgumentNotValidException ex) {
|
public ApiResponse<Void> handleMethodArgumentNotValidException(MethodArgumentNotValidException ex) { String message = ex.getBindingResult().getFieldError() != null
|
||||||
String message = ex.getBindingResult().getFieldError() != null
|
|
||||||
? ex.getBindingResult().getFieldError().getDefaultMessage()
|
? ex.getBindingResult().getFieldError().getDefaultMessage()
|
||||||
: "参数校验失败";
|
: "参数校验失败";
|
||||||
return ApiResponse.fail(message);
|
return ApiResponse.fail(message);
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
package com.nanri.aiimage.common.model.entity;
|
package com.nanri.aiimage.common.model.entity;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.annotation.FieldStrategy;
|
||||||
import com.baomidou.mybatisplus.annotation.IdType;
|
import com.baomidou.mybatisplus.annotation.IdType;
|
||||||
import com.baomidou.mybatisplus.annotation.TableField;
|
import com.baomidou.mybatisplus.annotation.TableField;
|
||||||
import com.baomidou.mybatisplus.annotation.TableId;
|
import com.baomidou.mybatisplus.annotation.TableId;
|
||||||
@@ -24,5 +25,13 @@ public class AdminUserEntity {
|
|||||||
private Long createdById;
|
private Long createdById;
|
||||||
@TableField("created_at")
|
@TableField("created_at")
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
|
/**
|
||||||
|
* 更新时间(V131 新增):由数据库维护(DEFAULT CURRENT_TIMESTAMP + ON UPDATE CURRENT_TIMESTAMP)。
|
||||||
|
*
|
||||||
|
* <p>禁止应用显式写:MySQL 在 UPDATE 语句显式给该列赋值时不会触发自动更新,
|
||||||
|
* 而本表写回多为 selectById → 改字段 → updateById(实体带着旧值),一旦写回就会冻结更新时间。
|
||||||
|
*/
|
||||||
|
@TableField(value = "updated_at", insertStrategy = FieldStrategy.NEVER, updateStrategy = FieldStrategy.NEVER)
|
||||||
|
private LocalDateTime updatedAt;
|
||||||
private String machine;
|
private String machine;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import io.jsonwebtoken.Claims;
|
|||||||
import jakarta.servlet.http.Cookie;
|
import jakarta.servlet.http.Cookie;
|
||||||
import jakarta.servlet.http.HttpServletRequest;
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.http.HttpHeaders;
|
import org.springframework.http.HttpHeaders;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
@@ -19,6 +20,7 @@ import java.nio.charset.StandardCharsets;
|
|||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
@Component
|
@Component
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class AdminAuthSupport {
|
public class AdminAuthSupport {
|
||||||
@@ -62,8 +64,30 @@ public class AdminAuthSupport {
|
|||||||
return user;
|
return user;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 当前用户必须是管理员或超级管理员,否则抛 403。 */
|
/**
|
||||||
public AdminUserEntity requireAdmin(HttpServletRequest request) {
|
* 解析当前请求 JWT 中**签名的**设备标识(deviceId claim);识别不出时返回空串。
|
||||||
|
*
|
||||||
|
* <p>无 token、token 过期/非法、内部令牌通道调用一律返回空串——调用方必须把空串
|
||||||
|
* 当作"来源不明"做保守判定,绝不据此放宽任何限制。只认签名 claim,不接受
|
||||||
|
* X-Device-Id 请求头(头由客户端可控,见 {@link DeviceSessionPolicy} 类注释)。</p>
|
||||||
|
*
|
||||||
|
* <p>本方法只做识别、不做鉴权,因此解析失败不抛异常,仅记日志后返回空串,
|
||||||
|
* 避免把匿名/内部调用直接升级成 401。</p>
|
||||||
|
*/
|
||||||
|
public String currentDeviceId(HttpServletRequest request) {
|
||||||
|
String token = resolveToken(request);
|
||||||
|
if (token == null || token.isBlank()) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return DeviceSessionPolicy.claimDeviceId(jwtService.parse(token));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[auth] 解析 token 取设备标识失败,按来源不明处理: {}", ex.getMessage());
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 当前用户必须是管理员或超级管理员,否则抛 403。 */ public AdminUserEntity requireAdmin(HttpServletRequest request) {
|
||||||
AdminUserEntity user = requireUser(request);
|
AdminUserEntity user = requireUser(request);
|
||||||
String role = currentRole(user);
|
String role = currentRole(user);
|
||||||
if (role == null) {
|
if (role == null) {
|
||||||
|
|||||||
+61
-6
@@ -13,10 +13,12 @@ import org.springframework.http.ResponseEntity;
|
|||||||
import org.springframework.http.client.SimpleClientHttpRequestFactory;
|
import org.springframework.http.client.SimpleClientHttpRequestFactory;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.util.StreamUtils;
|
import org.springframework.util.StreamUtils;
|
||||||
|
import org.springframework.web.client.ResourceAccessException;
|
||||||
import org.springframework.web.client.RestClient;
|
import org.springframework.web.client.RestClient;
|
||||||
import org.springframework.web.util.ContentCachingRequestWrapper;
|
import org.springframework.web.util.ContentCachingRequestWrapper;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.net.ConnectException;
|
||||||
import java.util.Enumeration;
|
import java.util.Enumeration;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
@@ -28,6 +30,11 @@ public class TaskOwnerForwardService {
|
|||||||
|
|
||||||
public static final String FORWARDED_HEADER = "X-AIIMAGE-Owner-Forwarded";
|
public static final String FORWARDED_HEADER = "X-AIIMAGE-Owner-Forwarded";
|
||||||
|
|
||||||
|
/** 连接类失败的重试次数(含首次)。对端滚动重启时通常几秒内即可恢复。 */
|
||||||
|
private static final int CONNECT_RETRY_TIMES = 3;
|
||||||
|
/** 第 n 次重试前的退避:1s、2s(总等待不超过 3s,不长时间占用请求线程)。 */
|
||||||
|
private static final long CONNECT_RETRY_BACKOFF_MILLIS = 1000L;
|
||||||
|
|
||||||
private static final Set<String> HOP_BY_HOP_HEADERS = Set.of(
|
private static final Set<String> HOP_BY_HOP_HEADERS = Set.of(
|
||||||
"connection",
|
"connection",
|
||||||
"keep-alive",
|
"keep-alive",
|
||||||
@@ -58,12 +65,60 @@ public class TaskOwnerForwardService {
|
|||||||
HttpHeaders headers = copyForwardHeaders(request, ex.getCurrentInstanceId());
|
HttpHeaders headers = copyForwardHeaders(request, ex.getCurrentInstanceId());
|
||||||
log.info("[instance-routing] forward {} taskId={} operation={} owner={} current={} url={}",
|
log.info("[instance-routing] forward {} taskId={} operation={} owner={} current={} url={}",
|
||||||
method, ex.getTaskId(), ex.getOperation(), ex.getOwnerInstanceId(), ex.getCurrentInstanceId(), url);
|
method, ex.getTaskId(), ex.getOperation(), ex.getOwnerInstanceId(), ex.getCurrentInstanceId(), url);
|
||||||
return restClient().method(method)
|
return forwardWithConnectRetry(method, url, headers, body, ex);
|
||||||
.uri(url)
|
}
|
||||||
.headers(target -> target.addAll(headers))
|
|
||||||
.body(body)
|
/**
|
||||||
.retrieve()
|
* 转发带连接级重试。
|
||||||
.toEntity(byte[].class);
|
*
|
||||||
|
* <p>对端实例在部署窗口(原地换 JAR + 两节点滚动重启)内会有几秒的 Connection refused。
|
||||||
|
* 连接都没建立起来说明请求没到达对端,此时重放是安全的;而读超时不重试——对端可能
|
||||||
|
* 已经在处理,盲目重放会造成重复提交。线上由此丢过用户提交的结果。
|
||||||
|
*/
|
||||||
|
private ResponseEntity<byte[]> forwardWithConnectRetry(HttpMethod method, String url,
|
||||||
|
HttpHeaders headers, byte[] body,
|
||||||
|
TaskOwnerMismatchException ex) {
|
||||||
|
RuntimeException lastError = null;
|
||||||
|
for (int attempt = 1; attempt <= CONNECT_RETRY_TIMES; attempt++) {
|
||||||
|
try {
|
||||||
|
return restClient().method(method)
|
||||||
|
.uri(url)
|
||||||
|
.headers(target -> target.addAll(headers))
|
||||||
|
.body(body)
|
||||||
|
.retrieve()
|
||||||
|
.toEntity(byte[].class);
|
||||||
|
} catch (ResourceAccessException accessError) {
|
||||||
|
if (!isConnectFailure(accessError)) {
|
||||||
|
throw accessError;
|
||||||
|
}
|
||||||
|
lastError = accessError;
|
||||||
|
log.warn("[instance-routing] 转发连接失败,第 {}/{} 次 url={} taskId={} 原因={}",
|
||||||
|
attempt, CONNECT_RETRY_TIMES, url, ex.getTaskId(), accessError.getMessage());
|
||||||
|
if (attempt < CONNECT_RETRY_TIMES) {
|
||||||
|
sleepQuietly(CONNECT_RETRY_BACKOFF_MILLIS * attempt);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw lastError;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static boolean isConnectFailure(Throwable error) {
|
||||||
|
Throwable cursor = error;
|
||||||
|
while (cursor != null) {
|
||||||
|
if (cursor instanceof ConnectException) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
cursor = cursor.getCause();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void sleepQuietly(long millis) {
|
||||||
|
try {
|
||||||
|
Thread.sleep(millis);
|
||||||
|
} catch (InterruptedException interrupted) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private String resolveUrl(TaskOwnerMismatchException ex, String path) {
|
private String resolveUrl(TaskOwnerMismatchException ex, String path) {
|
||||||
|
|||||||
@@ -0,0 +1,70 @@
|
|||||||
|
package com.nanri.aiimage.common.util;
|
||||||
|
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.function.Function;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 有界 LRU 缓存:容量超限时自动淘汰最久未使用的条目。
|
||||||
|
*
|
||||||
|
* <p>用于按「外部端点」缓存长生命周期资源(HttpClient / RestClient)。这类资源各自持有
|
||||||
|
* 连接池与 selector 线程,无界累积会持续泄漏线程与内存:代理端点每次提取往往是新的
|
||||||
|
* IP:port(jikip 提取),无上限的缓存只增不减。
|
||||||
|
*
|
||||||
|
* <p>淘汰时只从缓存移除引用,不做显式关闭:JDK 的 HttpClientImpl 注册了 Cleaner,
|
||||||
|
* 对象不可达后由 GC 回收并关闭其 selector 线程;显式关闭反而可能打断仍在途的请求。
|
||||||
|
*/
|
||||||
|
public final class BoundedLruCache<K, V> {
|
||||||
|
|
||||||
|
/** 默认容量:代理端点数量级远小于此,足够覆盖热点端点又不至于累积。 */
|
||||||
|
public static final int DEFAULT_MAX_SIZE = 64;
|
||||||
|
|
||||||
|
private final int maxSize;
|
||||||
|
private final Map<K, V> store;
|
||||||
|
|
||||||
|
public BoundedLruCache() {
|
||||||
|
this(DEFAULT_MAX_SIZE);
|
||||||
|
}
|
||||||
|
|
||||||
|
public BoundedLruCache(int maxSize) {
|
||||||
|
this.maxSize = Math.max(1, maxSize);
|
||||||
|
// accessOrder=true 使 get 也刷新顺序(真正的 LRU);synchronizedMap 保证其线程安全
|
||||||
|
this.store = Collections.synchronizedMap(new LinkedHashMap<>(16, 0.75f, true) {
|
||||||
|
@Override
|
||||||
|
protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
|
||||||
|
return size() > BoundedLruCache.this.maxSize;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 取缓存值,缺失时用 loader 计算并放入。
|
||||||
|
*
|
||||||
|
* <p>与 {@code ConcurrentHashMap.computeIfAbsent} 不同,此处不保证 loader 的原子性:
|
||||||
|
* 并发首次访问同一 key 时可能各自构造一次,随后其中一个覆盖另一个。对
|
||||||
|
* HttpClient/RestClient 这类构造廉价且幂等的资源可接受,换来的是锁粒度更小。
|
||||||
|
*/
|
||||||
|
public V computeIfAbsent(K key, Function<K, V> loader) {
|
||||||
|
V existing = store.get(key);
|
||||||
|
if (existing != null) {
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
V created = loader.apply(key);
|
||||||
|
store.put(key, created);
|
||||||
|
return created;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int size() {
|
||||||
|
return store.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 当前容量上限,供日志与测试断言使用。 */
|
||||||
|
public int maxSize() {
|
||||||
|
return maxSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void clear() {
|
||||||
|
store.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
|||||||
import com.nanri.aiimage.common.api.ApiResponse;
|
import com.nanri.aiimage.common.api.ApiResponse;
|
||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||||
|
import com.nanri.aiimage.common.security.DeviceSessionPolicy;
|
||||||
import jakarta.servlet.FilterChain;
|
import jakarta.servlet.FilterChain;
|
||||||
import jakarta.servlet.ServletException;
|
import jakarta.servlet.ServletException;
|
||||||
import jakarta.servlet.http.HttpServletRequest;
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
@@ -153,7 +154,14 @@ public class AdminApiGuardFilter extends OncePerRequestFilter {
|
|||||||
ApiResponse<Void> body = ex.getCode() == null
|
ApiResponse<Void> body = ex.getCode() == null
|
||||||
? ApiResponse.fail(ex.getMessage())
|
? ApiResponse.fail(ex.getMessage())
|
||||||
: ApiResponse.fail(ex.getCode(), ex.getMessage());
|
: ApiResponse.fail(ex.getCode(), ex.getMessage());
|
||||||
log.warn("[admin-guard] {} {} rejected: {}", request.getMethod(), request.getRequestURI(), ex.getMessage());
|
// 401(登录已过期)与被顶下线是前端定时轮询(/api/notifications/summary、/api/user-secrets 等)
|
||||||
|
// 的常态:线上单节点一天近 3000 条,会把真实业务错误淹没。与 GlobalExceptionHandler
|
||||||
|
// 的 isRoutineAuthNoise 同一口径降为 debug。
|
||||||
|
if (isRoutineAuthNoise(ex.getCode())) {
|
||||||
|
log.debug("[admin-guard] {} {} rejected: {}", request.getMethod(), request.getRequestURI(), ex.getMessage());
|
||||||
|
} else {
|
||||||
|
log.warn("[admin-guard] {} {} rejected: {}", request.getMethod(), request.getRequestURI(), ex.getMessage());
|
||||||
|
}
|
||||||
response.setStatus(HttpServletResponse.SC_OK);
|
response.setStatus(HttpServletResponse.SC_OK);
|
||||||
response.setContentType("application/json;charset=UTF-8");
|
response.setContentType("application/json;charset=UTF-8");
|
||||||
response.getWriter().write(objectMapper.writeValueAsString(body));
|
response.getWriter().write(objectMapper.writeValueAsString(body));
|
||||||
@@ -168,6 +176,12 @@ public class AdminApiGuardFilter extends OncePerRequestFilter {
|
|||||||
chain.doFilter(request, response);
|
chain.doFilter(request, response);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 登录态过期 / 被其他设备顶下线:前端轮询的常态噪声,不占 WARN。 */
|
||||||
|
private static boolean isRoutineAuthNoise(Integer code) {
|
||||||
|
return Integer.valueOf(401).equals(code)
|
||||||
|
|| Integer.valueOf(DeviceSessionPolicy.CODE_KICKED).equals(code);
|
||||||
|
}
|
||||||
|
|
||||||
/** 命中受保护前缀(/api/admin、/debug、自助接口、用户态工具前缀及其子路径)才进入鉴权,其余请求直接放行。 */
|
/** 命中受保护前缀(/api/admin、/debug、自助接口、用户态工具前缀及其子路径)才进入鉴权,其余请求直接放行。 */
|
||||||
private boolean isGuarded(String uri) {
|
private boolean isGuarded(String uri) {
|
||||||
if (matchesPrefix(uri, ADMIN_API_PREFIX) || matchesPrefix(uri, DEBUG_PREFIX)) {
|
if (matchesPrefix(uri, ADMIN_API_PREFIX) || matchesPrefix(uri, DEBUG_PREFIX)) {
|
||||||
|
|||||||
+11
-2
@@ -30,15 +30,24 @@ public class AppearancePatentProperties {
|
|||||||
private int llmFirstAttemptReadTimeoutMillis = 60000;
|
private int llmFirstAttemptReadTimeoutMillis = 60000;
|
||||||
private int llmBatchSize = 10;
|
private int llmBatchSize = 10;
|
||||||
/**
|
/**
|
||||||
* 批内行级并发数,默认等于批量大小
|
* 批内行级并发数。批次串行提交,每行串行发 2 个 LLM 请求,
|
||||||
|
* 故该值≈单任务对 LLM 网关的瞬时并发;默认与品牌检测同为 5,避免多任务并行时成倍放大。
|
||||||
*/
|
*/
|
||||||
private int llmRowConcurrency = 10;
|
private int llmRowConcurrency = 5;
|
||||||
/**
|
/**
|
||||||
* 每行每个 LLM 请求的重试次数(含首次)
|
* 每行每个 LLM 请求的重试次数(含首次)
|
||||||
*/
|
*/
|
||||||
private int llmRetryTimes = 3;
|
private int llmRetryTimes = 3;
|
||||||
private int staleTimeoutMinutes = 30;
|
private int staleTimeoutMinutes = 30;
|
||||||
private String staleFinalizeCron = "0 */2 * * * *";
|
private String staleFinalizeCron = "0 */2 * * * *";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 二次判死线(心跳正常但连续 N 分钟无结果分片上报)阈值,分钟;<=0 关闭本线。
|
||||||
|
* 既有判定以 Redis heartbeat stale 为主信号,但心跳随 Python HTTP 心跳每分钟刷新——
|
||||||
|
* 主线程卡死时心跳线程照发,任务永远不 stale(与生产 28131 同型缺口)。
|
||||||
|
* 本线改看 biz_task_scope_state.last_chunk_at(仅分片上传时刷新)。
|
||||||
|
*/
|
||||||
|
private int noResultUploadTimeoutMinutes = 180;
|
||||||
/**
|
/**
|
||||||
* 末尾不足一批的数据等待该时长后强制提交检测。
|
* 末尾不足一批的数据等待该时长后强制提交检测。
|
||||||
* Python 回传超时后会直接封口;这个配置处理 Python 慢回传但仍未超时的零头批次。
|
* Python 回传超时后会直接封口;这个配置处理 Python 慢回传但仍未超时的零头批次。
|
||||||
|
|||||||
@@ -10,10 +10,27 @@ public class BrandCheckProperties {
|
|||||||
private String path = "/brand_check";
|
private String path = "/brand_check";
|
||||||
private String token = "";
|
private String token = "";
|
||||||
private String defaultStrategy = "Terms";
|
private String defaultStrategy = "Terms";
|
||||||
/** 单品牌商标查询的重试次数(含首次),连续失败超过该次数才判定为查询失败。 */
|
/**
|
||||||
private int retryTimes = 3;
|
* 单品牌商标查询的重试次数(含首次),连续失败超过该次数才判定为查询失败。
|
||||||
/** 每次查询失败后到下一次重试前的等待毫秒数。 */
|
* 原为 3:16890 偶发限流几秒内即恢复,3 次(前两次间隔各 1s)恢复不了就把结论
|
||||||
|
* 写成「查询失败」,对客户是硬伤;2026-09-14 与客户端品牌一致提到 10 次。
|
||||||
|
*/
|
||||||
|
private int retryTimes = 10;
|
||||||
|
/** 每次查询失败后到下一次重试前的等待毫秒数(基准值,按重试轮次递增)。 */
|
||||||
private int retryIntervalMillis = 1000;
|
private int retryIntervalMillis = 1000;
|
||||||
|
/**
|
||||||
|
* 单次重试等待的上限毫秒数。等待按 retryIntervalMillis × 第几次重试 递增后封顶,
|
||||||
|
* 10 次尝试合计约 45s,既能覆盖限流窗口又不会让单个品牌长时间占住查询线程。
|
||||||
|
*/
|
||||||
|
private int retryMaxIntervalMillis = 10000;
|
||||||
|
/**
|
||||||
|
* 一次批量检查的总耗时上限(毫秒):整批用尽后不再重试,剩余品牌按「查询失败」收尾。
|
||||||
|
* 上限的意义不是省时间,而是给「分片回传」这类同步调用方一个时延上界——上游卡死时
|
||||||
|
* 单品牌 10 次重试曾把一次回传拖到 103.5 秒(taskId 28599),客户端重试预算耗尽后
|
||||||
|
* 中止了整个采集。首轮请求始终执行,故上游只是略慢时不会误降级。
|
||||||
|
* 设为 0 或负数表示不限制。
|
||||||
|
*/
|
||||||
|
private int totalTimeoutMillis = 90000;
|
||||||
private int connectTimeoutMillis = 10000;
|
private int connectTimeoutMillis = 10000;
|
||||||
private int readTimeoutMillis = 60000;
|
private int readTimeoutMillis = 60000;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,4 +10,11 @@ public class BrandProgressProperties {
|
|||||||
private long failedTtlHours = 2;
|
private long failedTtlHours = 2;
|
||||||
private long heartbeatTimeoutMinutes = 15;
|
private long heartbeatTimeoutMinutes = 15;
|
||||||
private String staleCheckCron = "0 */2 * * * *";
|
private String staleCheckCron = "0 */2 * * * *";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 二次判死线(心跳正常但连续 N 分钟无结果上报)阈值,分钟;<=0 关闭本线。
|
||||||
|
* 既有心跳线以 updated_at/last_heartbeat_at 陈旧为判据,而前端心跳会持续刷新它们——
|
||||||
|
* 主线程卡死时心跳线程照发,任务永远命不中。本线改看结果上报时写入的 last_result_at。
|
||||||
|
*/
|
||||||
|
private long noResultUploadTimeoutMinutes = 180;
|
||||||
}
|
}
|
||||||
|
|||||||
+13
@@ -45,6 +45,19 @@ public class DeleteBrandProgressProperties {
|
|||||||
*/
|
*/
|
||||||
private long withdrawStaleTimeoutMinutes = 30;
|
private long withdrawStaleTimeoutMinutes = 30;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 「心跳正常但连续 N 分钟无结果分片上报」的二次判死阈值(分钟),默认 3 小时。
|
||||||
|
*
|
||||||
|
* <p>既有各模块心跳线的候选条件是 updated_at 陈旧,而客户端心跳会持续刷新 updated_at——
|
||||||
|
* 主线程卡死时心跳线程照发,任务永远命不中(生产 28131 卡死 12h+ 仍 RUNNING)。
|
||||||
|
* 本线改用 biz_task_scope_state.last_chunk_at(只随结果分片上报刷新)作判据,
|
||||||
|
* 从未上报(无 scope 行或全 NULL)的任务跳过,避免误杀首批较慢的正常任务。
|
||||||
|
*/
|
||||||
|
private long noResultUploadTimeoutMinutes = 180;
|
||||||
|
|
||||||
|
/** 二次判死线开关:false = 整段不扫描(观察期与回滚用,改环境变量即生效)。 */
|
||||||
|
private boolean noResultUploadCheckEnabled = true;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 组装结果文件时允许缺失的最大分片数:缺失不超过该值时降级组装(缺失行状态写"未回传"),
|
* 组装结果文件时允许缺失的最大分片数:缺失不超过该值时降级组装(缺失行状态写"未回传"),
|
||||||
* 保证任务不因少量分片丢失整体白跑;超出时任务失败,但仍会在重试耗尽后尝试
|
* 保证任务不因少量分片丢失整体白跑;超出时任务失败,但仍会在重试耗尽后尝试
|
||||||
|
|||||||
@@ -0,0 +1,38 @@
|
|||||||
|
package com.nanri.aiimage.config;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 设备日志对象存储配置:指向主机B 独立部署的 MinIO 实例(非业务 MinIO)。
|
||||||
|
*
|
||||||
|
* <p>日志体积大、只保留 7 天,独立实例便于单独设生命周期规则与容量管理,
|
||||||
|
* 不挤占业务桶(nanri-ai-images 等)。endpoint 为空时上报接口直接失败,
|
||||||
|
* 不做静默回退(避免日志悄悄落到其他存储上而无人知情)。
|
||||||
|
*/
|
||||||
|
@Data
|
||||||
|
@ConfigurationProperties(prefix = "aiimage.device-log-oss")
|
||||||
|
public class DeviceLogOssProperties {
|
||||||
|
|
||||||
|
private String endpoint;
|
||||||
|
private String accessKeyId;
|
||||||
|
private String accessKeySecret;
|
||||||
|
private String bucket;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 日志保留天数:查询侧按此过滤(早于今天的 N-1 天不展示),
|
||||||
|
* 对象过期由 MinIO 桶生命周期规则在部署时同步设置(两侧口径保持一致)。
|
||||||
|
*/
|
||||||
|
private Integer retentionDays;
|
||||||
|
|
||||||
|
public boolean configured() {
|
||||||
|
return endpoint != null && !endpoint.isBlank()
|
||||||
|
&& accessKeyId != null && !accessKeyId.isBlank()
|
||||||
|
&& accessKeySecret != null && !accessKeySecret.isBlank()
|
||||||
|
&& bucket != null && !bucket.isBlank();
|
||||||
|
}
|
||||||
|
|
||||||
|
public int retentionDaysOrDefault() {
|
||||||
|
return retentionDays == null || retentionDays < 1 ? 7 : retentionDays;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,5 +1,7 @@
|
|||||||
package com.nanri.aiimage.config;
|
package com.nanri.aiimage.config;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.util.BoundedLruCache;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.http.client.ClientHttpRequestFactory;
|
import org.springframework.http.client.ClientHttpRequestFactory;
|
||||||
import org.springframework.http.client.JdkClientHttpRequestFactory;
|
import org.springframework.http.client.JdkClientHttpRequestFactory;
|
||||||
|
|
||||||
@@ -14,8 +16,6 @@ import java.net.http.HttpClient;
|
|||||||
import java.net.http.HttpRequest;
|
import java.net.http.HttpRequest;
|
||||||
import java.net.http.HttpResponse;
|
import java.net.http.HttpResponse;
|
||||||
import java.time.Duration;
|
import java.time.Duration;
|
||||||
import java.util.Map;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Task 77:外部 HTTP 客户端统一连接复用池。
|
* Task 77:外部 HTTP 客户端统一连接复用池。
|
||||||
@@ -24,6 +24,7 @@ import java.util.concurrent.ConcurrentHashMap;
|
|||||||
* 每次请求都重新建连。各客户端按自身超时创建独立的
|
* 每次请求都重新建连。各客户端按自身超时创建独立的
|
||||||
* JdkClientHttpRequestFactory(共享底层连接池),RestClient 单例懒加载。
|
* JdkClientHttpRequestFactory(共享底层连接池),RestClient 单例懒加载。
|
||||||
*/
|
*/
|
||||||
|
@Slf4j
|
||||||
public class HttpClientPool {
|
public class HttpClientPool {
|
||||||
|
|
||||||
private static volatile HttpClient sharedHttpClient;
|
private static volatile HttpClient sharedHttpClient;
|
||||||
@@ -103,8 +104,12 @@ public class HttpClientPool {
|
|||||||
public static ClientHttpRequestFactory requestFactory(int readTimeoutMillis, String proxyUrl) {
|
public static ClientHttpRequestFactory requestFactory(int readTimeoutMillis, String proxyUrl) {
|
||||||
long safeReadTimeout = Math.max(1L, readTimeoutMillis);
|
long safeReadTimeout = Math.max(1L, readTimeoutMillis);
|
||||||
long callTimeout = configuredCallTimeoutMillis;
|
long callTimeout = configuredCallTimeoutMillis;
|
||||||
if (callTimeout > 0L) {
|
if (callTimeout > 0L && safeReadTimeout > callTimeout) {
|
||||||
safeReadTimeout = Math.min(safeReadTimeout, callTimeout);
|
// 读超时以调用方显式值为准,不再被全局 call-timeout 截断:
|
||||||
|
// LLM 长思考配的是 180s(llm-read-timeout-millis),曾被静默压到 90s,
|
||||||
|
// 导致请求在 90s 被掐断 → 上层重试 → 付费网关二次计费(2026-09-15 修复)。
|
||||||
|
// 各调用方的超时已由各自的 HttpConfigResolver 钳制,此处不再二次收敛。
|
||||||
|
log.debug("读超时 {}ms 超过全局 call-timeout {}ms,按调用方显式值生效", safeReadTimeout, callTimeout);
|
||||||
}
|
}
|
||||||
JdkClientHttpRequestFactory factory =
|
JdkClientHttpRequestFactory factory =
|
||||||
new JdkClientHttpRequestFactory(httpClientFor(proxyUrl));
|
new JdkClientHttpRequestFactory(httpClientFor(proxyUrl));
|
||||||
@@ -166,5 +171,6 @@ public class HttpClientPool {
|
|||||||
private record ProxyEndpoint(String host, int port, String userInfo) {
|
private record ProxyEndpoint(String host, int port, String userInfo) {
|
||||||
}
|
}
|
||||||
|
|
||||||
private static final Map<ProxyEndpoint, HttpClient> PROXY_CLIENTS = new ConcurrentHashMap<>();
|
private static final BoundedLruCache<ProxyEndpoint, HttpClient> PROXY_CLIENTS =
|
||||||
|
new BoundedLruCache<>(BoundedLruCache.DEFAULT_MAX_SIZE);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -41,6 +41,38 @@ public class NotificationProperties {
|
|||||||
/** jikip 代理接口探测开关(复用余额查询接口判活,使用 user-secret 的 jikip 配置)。 */
|
/** jikip 代理接口探测开关(复用余额查询接口判活,使用 user-secret 的 jikip 配置)。 */
|
||||||
private boolean jikipProbeEnabled = true;
|
private boolean jikipProbeEnabled = true;
|
||||||
|
|
||||||
|
/** 麦象(18960 任务调度)异常扫描开关:任务停滞/失败/队列积压 → 管理员通知。 */
|
||||||
|
private boolean maixiangScanEnabled = true;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 麦象后台接口令牌(18960 console token)。留空=跳过麦象异常扫描——
|
||||||
|
* 该令牌与「跟价任务 API 地址」(priceTrackApiUrl) 一起构成后台只读接口的访问凭据。
|
||||||
|
*/
|
||||||
|
private String maixiangConsoleToken = "";
|
||||||
|
|
||||||
|
/** 麦象批量任务停滞阈值(分钟):status=0/1 且超过该时长无更新视为卡住。 */
|
||||||
|
private int maixiangStuckMinutes = 60;
|
||||||
|
|
||||||
|
/** 麦象单任务滞留阈值(分钟):创建超时仍未完成(status=0/1)视为滞留/无人消费。 */
|
||||||
|
private int maixiangSingleStuckMinutes = 30;
|
||||||
|
|
||||||
|
/** 麦象任务失败告警阈值(条):近 30 分钟窗口内失败数达到该值才告警。 */
|
||||||
|
private int maixiangFailMinCount = 1;
|
||||||
|
|
||||||
|
/** 麦象队列积压阈值(条):task:queue 待处理数达到该值告警。 */
|
||||||
|
private int maixiangQueuePendingThreshold = 300;
|
||||||
|
|
||||||
|
/** 麦象队列积压阈值(条):task:processing 处理中数达到该值告警。 */
|
||||||
|
private int maixiangQueueProcessingThreshold = 100;
|
||||||
|
|
||||||
/** 已读通知保留天数(超期自动清理),默认 90 天。 */
|
/** 已读通知保留天数(超期自动清理),默认 90 天。 */
|
||||||
private int readRetentionDays = 90;
|
private int readRetentionDays = 90;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 未读通知保留天数,默认 180 天(比已读长一倍)。
|
||||||
|
*
|
||||||
|
* <p>未读通知此前永不清理:不看铃铛的用户会无限累积。保留期给得更宽,
|
||||||
|
* 是因为未读意味着"用户可能还没看到",但也不能永远留着。
|
||||||
|
*/
|
||||||
|
private int unreadRetentionDays = 180;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
|
|||||||
import org.springframework.context.annotation.Configuration;
|
import org.springframework.context.annotation.Configuration;
|
||||||
|
|
||||||
@Configuration
|
@Configuration
|
||||||
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, BrandCheckProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, TaskImageCacheCleanupProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class, ImageVideoProperties.class, InstanceRoutingProperties.class, CapacityPlanProperties.class, UserSecretProperties.class, NotificationProperties.class})
|
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, BrandCheckProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, TaskImageCacheCleanupProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class, ImageVideoProperties.class, InstanceRoutingProperties.class, CapacityPlanProperties.class, UserSecretProperties.class, NotificationProperties.class, DeviceLogOssProperties.class})
|
||||||
public class PropertiesConfig {
|
public class PropertiesConfig {
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,6 +23,14 @@ public class SimilarAsinProperties {
|
|||||||
private int staleTimeoutMinutes = 30;
|
private int staleTimeoutMinutes = 30;
|
||||||
private String staleFinalizeCron = "0 */2 * * * *";
|
private String staleFinalizeCron = "0 */2 * * * *";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 二次判死线(心跳正常但连续 N 分钟无结果分片上报)阈值,分钟;<=0 关闭本线。
|
||||||
|
* 既有判定以 Redis heartbeat stale 为主信号,但心跳随 Python HTTP 心跳每分钟刷新——
|
||||||
|
* 主线程卡死时心跳线程照发,任务永远不 stale(与生产 28131 同型缺口)。
|
||||||
|
* 本线改看 biz_task_scope_state.last_chunk_at(仅分片上传时刷新)。
|
||||||
|
*/
|
||||||
|
private int noResultUploadTimeoutMinutes = 180;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 末尾零头 batch 的强制 flush 阈值(分钟):当不足 llmBatchSize 的零头 row
|
* 末尾零头 batch 的强制 flush 阈值(分钟):当不足 llmBatchSize 的零头 row
|
||||||
* 长时间挂着(Python 慢回传)时触发提交。
|
* 长时间挂着(Python 慢回传)时触发提交。
|
||||||
|
|||||||
+20
-1
@@ -29,7 +29,14 @@ public class TransientStorageProperties {
|
|||||||
*/
|
*/
|
||||||
private long maxTotalConcurrentOperations = 0;
|
private long maxTotalConcurrentOperations = 0;
|
||||||
private long acquirePermitTimeoutMillis = 2000;
|
private long acquirePermitTimeoutMillis = 2000;
|
||||||
private long baseRetryDelayMillis = 500;
|
/**
|
||||||
|
* 首次重试前的基础退避。
|
||||||
|
*
|
||||||
|
* <p>线上高频的重试诱因是 `unexpected end of stream`——那是**立即失败**(连接被 RustFS
|
||||||
|
* 重置后 OkHttp 读响应即报错),不是等超时,所以 500ms 基本是白等:每天上千次累计十几分钟。
|
||||||
|
* 降到 200ms 保留退避语义(真遇到服务端过载仍会退让),又不至于让用户等太久。
|
||||||
|
*/
|
||||||
|
private long baseRetryDelayMillis = 200;
|
||||||
private long maxRetryDelayMillis = 5000;
|
private long maxRetryDelayMillis = 5000;
|
||||||
private long retryJitterMillis = 250;
|
private long retryJitterMillis = 250;
|
||||||
private long failureWindowSeconds = 60;
|
private long failureWindowSeconds = 60;
|
||||||
@@ -37,7 +44,19 @@ public class TransientStorageProperties {
|
|||||||
private long failureCooldownMillis = 10000;
|
private long failureCooldownMillis = 10000;
|
||||||
private int dispatcherMaxRequests = 56;
|
private int dispatcherMaxRequests = 56;
|
||||||
private int dispatcherMaxRequestsPerHost = 56;
|
private int dispatcherMaxRequestsPerHost = 56;
|
||||||
|
/**
|
||||||
|
* 空闲连接保留数。
|
||||||
|
*
|
||||||
|
* <p>2026-09-17 曾试过设 0(彻底不复用)来验证"unexpected end of stream 是复用死连接导致的"
|
||||||
|
* 这一假设——**实测照旧失败**(新容器起来后第一次请求就中招)。至此已排除公网链路、
|
||||||
|
* keepAlive 过长、连接复用三项;用 mc 并发压 200 个小对象也全部成功,说明服务端没问题。
|
||||||
|
* 剩余方向指向 MinIO Java SDK / OkHttp 与 RustFS 的协议细节,故恢复默认的连接复用。
|
||||||
|
*/
|
||||||
private int connectionPoolMaxIdle = 5;
|
private int connectionPoolMaxIdle = 5;
|
||||||
|
/**
|
||||||
|
* 空闲连接在池里的保留时长。曾由 300000 调到 30000 试图减少 unexpected end of stream,
|
||||||
|
* 实测无改善(该现象与连接复用无关,见 {@link #connectionPoolMaxIdle} 的排查记录),故恢复原值。
|
||||||
|
*/
|
||||||
private long connectionPoolKeepAliveMillis = 300000;
|
private long connectionPoolKeepAliveMillis = 300000;
|
||||||
private long warnPayloadBytes = 5L * 1024 * 1024;
|
private long warnPayloadBytes = 5L * 1024 * 1024;
|
||||||
private long maxPayloadBytes = 50L * 1024 * 1024;
|
private long maxPayloadBytes = 50L * 1024 * 1024;
|
||||||
|
|||||||
@@ -22,6 +22,13 @@ public class UserSecretProperties {
|
|||||||
/** 单轮巡检时间预算(分钟),超时中断本轮。 */
|
/** 单轮巡检时间预算(分钟),超时中断本轮。 */
|
||||||
private int checkBudgetMinutes = 20;
|
private int checkBudgetMinutes = 20;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 检测请求使用的 LLM 模型:独立于业务任务模型(业务用 gemini-3.8-flash 等),
|
||||||
|
* 选便宜的可用模型,只验证密钥有效性与链路连通,降低每次检测与巡检的成本。
|
||||||
|
* 用 lite 而非 mini:mini 在中继分组下无可用渠道(503 model_not_found),实测 lite 可路由。
|
||||||
|
*/
|
||||||
|
private String checkModel = "doubao-seed-2-0-lite-260215";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 检测出口代理提取链接(选配):配置后检测请求优先经该代理 IP 发出,
|
* 检测出口代理提取链接(选配):配置后检测请求优先经该代理 IP 发出,
|
||||||
* 代理不可用时自动回退直连;留空则全部直连。
|
* 代理不可用时自动回退直连;留空则全部直连。
|
||||||
|
|||||||
+3
-1
@@ -76,8 +76,10 @@ public class AdminConsoleController {
|
|||||||
@Operation(summary = "当前登录管理员的可见后台菜单树")
|
@Operation(summary = "当前登录管理员的可见后台菜单树")
|
||||||
public ApiResponse<Map<String, Object>> currentUserMenus(HttpServletRequest request) {
|
public ApiResponse<Map<String, Object>> currentUserMenus(HttpServletRequest request) {
|
||||||
AdminUserEntity operator = adminAuthSupport.requireAdminOrInternal(request);
|
AdminUserEntity operator = adminAuthSupport.requireAdminOrInternal(request);
|
||||||
|
// 补全祖先分组:部分授权用户(只授权了子页面)也要看到「一级分组 + 子页面」层级,
|
||||||
|
// 与超管的菜单组织顺序一致;分组节点无页面路由,不构成权限扩展。
|
||||||
List<PermissionMenuItemVo> menus = permissionMenuService.getUserColumnPermissions(
|
List<PermissionMenuItemVo> menus = permissionMenuService.getUserColumnPermissions(
|
||||||
operator, operator.getId(), PermissionMenuService.MENU_TYPE_ADMIN);
|
operator, operator.getId(), PermissionMenuService.MENU_TYPE_ADMIN, true);
|
||||||
menus = AdminMenuTreeBuilder.filterByMenuType(menus, PermissionMenuService.MENU_TYPE_ADMIN);
|
menus = AdminMenuTreeBuilder.filterByMenuType(menus, PermissionMenuService.MENU_TYPE_ADMIN);
|
||||||
List<Map<String, Object>> items = AdminMenuTreeBuilder.toMapList(AdminMenuTreeBuilder.build(menus));
|
List<Map<String, Object>> items = AdminMenuTreeBuilder.toMapList(AdminMenuTreeBuilder.build(menus));
|
||||||
return ApiResponse.success(Map.of("items", items));
|
return ApiResponse.success(Map.of("items", items));
|
||||||
|
|||||||
+2
@@ -16,6 +16,8 @@ public class AdminUserItemVo {
|
|||||||
private String creatorUsername;
|
private String creatorUsername;
|
||||||
@JsonProperty("created_at")
|
@JsonProperty("created_at")
|
||||||
private String createdAt;
|
private String createdAt;
|
||||||
|
@JsonProperty("updated_at")
|
||||||
|
private String updatedAt;
|
||||||
@JsonProperty("pinyin_abbr")
|
@JsonProperty("pinyin_abbr")
|
||||||
private String pinyinAbbr;
|
private String pinyinAbbr;
|
||||||
}
|
}
|
||||||
|
|||||||
+2
@@ -338,6 +338,8 @@ public class AdminUserService {
|
|||||||
vo.setCreatorUsername(creatorId == null ? "" : creatorMap.getOrDefault(creatorId, ""));
|
vo.setCreatorUsername(creatorId == null ? "" : creatorMap.getOrDefault(creatorId, ""));
|
||||||
LocalDateTime createdAt = entity.getCreatedAt();
|
LocalDateTime createdAt = entity.getCreatedAt();
|
||||||
vo.setCreatedAt(createdAt == null ? "" : createdAt.format(CREATED_AT_FORMATTER));
|
vo.setCreatedAt(createdAt == null ? "" : createdAt.format(CREATED_AT_FORMATTER));
|
||||||
|
LocalDateTime updatedAt = entity.getUpdatedAt();
|
||||||
|
vo.setUpdatedAt(updatedAt == null ? "" : updatedAt.format(CREATED_AT_FORMATTER));
|
||||||
vo.setPinyinAbbr(PinyinAbbrUtil.abbr(entity.getUsername()));
|
vo.setPinyinAbbr(PinyinAbbrUtil.abbr(entity.getUsername()));
|
||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
|
|||||||
+45
@@ -0,0 +1,45 @@
|
|||||||
|
package com.nanri.aiimage.modules.appconfig.controller;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.api.ApiResponse;
|
||||||
|
import com.nanri.aiimage.modules.appconfig.service.KdFlowService;
|
||||||
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
|
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||||
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestBody;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 工作台「开店流程」模块访问密码校验(公开接口,密码本身就是凭据,不额外要求登录态)。
|
||||||
|
*
|
||||||
|
* <p>客户端只在用户点开「开店流程」分组时调用一次;返回体只给 ok 与中文提示,
|
||||||
|
* 不回显服务端配置的密码。
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@RestController
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Tag(name = "开店流程访问校验", description = "工作台「开店流程」模块访问密码的服务端校验")
|
||||||
|
public class KdFlowController {
|
||||||
|
|
||||||
|
private final KdFlowService kdFlowService;
|
||||||
|
|
||||||
|
@PostMapping("/api/kd-flow/verify")
|
||||||
|
@Operation(summary = "校验开店流程访问密码",
|
||||||
|
description = "密码存 app_config.kd_flow_password;改密码只需 UPDATE 该行,客户端无需重新发布")
|
||||||
|
public ApiResponse<Map<String, Object>> verify(@RequestBody(required = false) Map<String, String> body,
|
||||||
|
HttpServletRequest request) {
|
||||||
|
String input = body == null ? null : body.get("password");
|
||||||
|
boolean ok = kdFlowService.matches(input);
|
||||||
|
// 只记输入长度与结果,绝不回显密码本身
|
||||||
|
log.info("[开店流程] 校验请求 remoteAddr={} 输入为空={} 结果={}",
|
||||||
|
request.getRemoteAddr(), input == null || input.isBlank(), ok ? "通过" : "拒绝");
|
||||||
|
if (!ok) {
|
||||||
|
return ApiResponse.fail("密码错误");
|
||||||
|
}
|
||||||
|
return ApiResponse.success("验证通过", Map.of("ok", true));
|
||||||
|
}
|
||||||
|
}
|
||||||
+9
@@ -0,0 +1,9 @@
|
|||||||
|
package com.nanri.aiimage.modules.appconfig.mapper;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||||
|
import com.nanri.aiimage.modules.appconfig.model.entity.AppConfigEntity;
|
||||||
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
|
|
||||||
|
@Mapper
|
||||||
|
public interface AppConfigMapper extends BaseMapper<AppConfigEntity> {
|
||||||
|
}
|
||||||
+32
@@ -0,0 +1,32 @@
|
|||||||
|
package com.nanri.aiimage.modules.appconfig.model.entity;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.annotation.IdType;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableId;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableName;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 通用应用配置(键值)。首个用途:工作台「开店流程」模块访问密码(key = kd_flow_password)。
|
||||||
|
* <p>只放这类低价值、需要"改一行即生效"的口令,不放密钥类敏感配置。
|
||||||
|
*/
|
||||||
|
@Data
|
||||||
|
@TableName("app_config")
|
||||||
|
public class AppConfigEntity {
|
||||||
|
|
||||||
|
@TableId(type = IdType.AUTO)
|
||||||
|
private Long id;
|
||||||
|
|
||||||
|
/** 配置键(唯一) */
|
||||||
|
private String configKey;
|
||||||
|
|
||||||
|
/** 配置值 */
|
||||||
|
private String configValue;
|
||||||
|
|
||||||
|
/** 说明 */
|
||||||
|
private String remark;
|
||||||
|
|
||||||
|
/** 更新时间,由数据库 CURRENT_TIMESTAMP 维护 */
|
||||||
|
private LocalDateTime updatedAt;
|
||||||
|
}
|
||||||
+48
@@ -0,0 +1,48 @@
|
|||||||
|
package com.nanri.aiimage.modules.appconfig.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.nanri.aiimage.modules.appconfig.mapper.AppConfigMapper;
|
||||||
|
import com.nanri.aiimage.modules.appconfig.model.entity.AppConfigEntity;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 工作台「开店流程」模块访问密码的服务端校验。
|
||||||
|
*
|
||||||
|
* <p>此前密码写死在客户端源码(KD_FLOW_PASSWORD),改密码必须重新打包装包发给全部用户;
|
||||||
|
* 改由服务端比对后,改密码只需 UPDATE app_config 一行(key = kd_flow_password)。
|
||||||
|
*
|
||||||
|
* <p>不缓存:调用频次极低(用户点一次分组头一次),且改密码后应立即生效。
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class KdFlowService {
|
||||||
|
|
||||||
|
/** app_config 中存放开店流程访问密码的键名 */
|
||||||
|
public static final String PASSWORD_KEY = "kd_flow_password";
|
||||||
|
|
||||||
|
private final AppConfigMapper appConfigMapper;
|
||||||
|
|
||||||
|
/** 读取服务端配置的密码;未配置返回 null。 */
|
||||||
|
public String configuredPassword() {
|
||||||
|
AppConfigEntity row = appConfigMapper.selectOne(new LambdaQueryWrapper<AppConfigEntity>()
|
||||||
|
.eq(AppConfigEntity::getConfigKey, PASSWORD_KEY)
|
||||||
|
.last("LIMIT 1"));
|
||||||
|
return row == null ? null : row.getConfigValue();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 校验用户输入的密码。未配置密码时一律判失败(宁可锁死也不放行)。 */
|
||||||
|
public boolean matches(String input) {
|
||||||
|
String expect = configuredPassword();
|
||||||
|
if (expect == null || expect.isBlank()) {
|
||||||
|
log.warn("[开店流程] app_config 未配置 {},本次校验一律判失败", PASSWORD_KEY);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
String actual = input == null ? "" : input.trim();
|
||||||
|
boolean ok = expect.equals(actual);
|
||||||
|
log.info("[开店流程] 服务端校验 输入长度={} 结果={}", actual.length(), ok ? "通过" : "不通过");
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
}
|
||||||
+2
-2
@@ -110,8 +110,8 @@ public class AppearancePatentController {
|
|||||||
|
|
||||||
@PostMapping("/tasks/progress/light")
|
@PostMapping("/tasks/progress/light")
|
||||||
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
|
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
|
||||||
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt/rowsVersion),"
|
||||||
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
+ "不返回行明细内容(仅多一次结果行版本聚合查询),响应体更小;旧 progress/batch 端点保留不动。")
|
||||||
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||||
// 透传 userId:此前传 null = 不过滤归属,匿名遍历 taskId 即可读他人任务状态(2026-09 审查)
|
// 透传 userId:此前传 null = 不过滤归属,匿名遍历 taskId 即可读他人任务状态(2026-09 审查)
|
||||||
return ApiResponse.success(service.progressLight(request.getTaskIds(), request.getUserId()));
|
return ApiResponse.success(service.progressLight(request.getTaskIds(), request.getUserId()));
|
||||||
|
|||||||
+60
@@ -0,0 +1,60 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParsedPayloadDto;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedGroupVo;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.spi.ClientTaskPullSpi;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 外观专利的客户端兜底拉取实现。
|
||||||
|
*
|
||||||
|
* <p>Python 消费端需要 groups(解析分组,页面上是现拉 /queue-payload 再入队),
|
||||||
|
* 这里直接复用同一个 service 方法;payload 与页面保持一致(含 prompt / api_key)。
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class AppearancePatentTaskPullSpiImpl implements ClientTaskPullSpi {
|
||||||
|
|
||||||
|
private static final String QUEUE_TYPE = "appearance-patent-run";
|
||||||
|
|
||||||
|
private final AppearancePatentTaskService taskService;
|
||||||
|
private final AppearancePatentTaskCacheService taskCacheService;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String moduleType() {
|
||||||
|
return AppearancePatentTaskService.MODULE_TYPE;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<String, Object> buildQueuePayload(FileTaskEntity task) {
|
||||||
|
AppearancePatentParsedPayloadDto payload = taskService.queuePayload(task.getId(), task.getUserId());
|
||||||
|
List<AppearancePatentParsedGroupVo> groups = payload.getGroups() == null ? List.of() : payload.getGroups();
|
||||||
|
if (groups.isEmpty()) {
|
||||||
|
// 空 groups 在 Python 侧会被静默跳过("groups/rows is empty, skip"),宁可在服务端直接判失败
|
||||||
|
log.warn("[appearance-patent] 兜底拉取失败:解析分组为空 taskId={}", task.getId());
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
Map<String, Object> data = new LinkedHashMap<>();
|
||||||
|
data.put("taskId", task.getId());
|
||||||
|
data.put("user_id", task.getUserId());
|
||||||
|
data.put("prompt", payload.getAiPrompt());
|
||||||
|
data.put("api_key", payload.getApiKey());
|
||||||
|
data.put("groups", groups);
|
||||||
|
log.info("[appearance-patent] 兜底载荷已组装 taskId={} groups={}", task.getId(), groups.size());
|
||||||
|
return Map.of("type", QUEUE_TYPE, "data", data);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onClaimed(FileTaskEntity task) {
|
||||||
|
// 对齐 activate:刷新模块缓存心跳,让页面立刻看到 RUNNING
|
||||||
|
taskCacheService.touchTaskHeartbeat(task.getId());
|
||||||
|
}
|
||||||
|
}
|
||||||
+244
-15
@@ -40,6 +40,7 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
|||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
@@ -68,6 +69,8 @@ import org.springframework.stereotype.Service;
|
|||||||
import org.springframework.transaction.PlatformTransactionManager;
|
import org.springframework.transaction.PlatformTransactionManager;
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
import org.springframework.transaction.TransactionDefinition;
|
import org.springframework.transaction.TransactionDefinition;
|
||||||
|
import org.springframework.transaction.support.TransactionSynchronization;
|
||||||
|
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||||
import org.springframework.transaction.support.TransactionTemplate;
|
import org.springframework.transaction.support.TransactionTemplate;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
@@ -82,6 +85,7 @@ import java.util.ArrayList;
|
|||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.LinkedHashSet;
|
import java.util.LinkedHashSet;
|
||||||
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
@@ -280,12 +284,20 @@ public class AppearancePatentTaskService {
|
|||||||
throw new BusinessException("任务不存在");
|
throw new BusinessException("任务不存在");
|
||||||
}
|
}
|
||||||
ensureTaskOwnedByCurrentInstance(task, "activate");
|
ensureTaskOwnedByCurrentInstance(task, "activate");
|
||||||
if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) {
|
// 只允许 PENDING→RUNNING(条件更新):与客户端「兜底拉取」的原子认领互斥,
|
||||||
|
// 谁先翻转谁执行,避免页面与客户端重复执行同一任务
|
||||||
|
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||||
|
.eq(FileTaskEntity::getId, taskId)
|
||||||
|
.eq(FileTaskEntity::getStatus, STATUS_PENDING)
|
||||||
|
.set(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||||
|
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
||||||
|
if (updated == 0) {
|
||||||
|
FileTaskEntity latest = fileTaskMapper.selectById(taskId);
|
||||||
|
if (latest != null && STATUS_RUNNING.equals(latest.getStatus())) {
|
||||||
|
throw new BusinessException("任务已在执行中(可能已由客户端自动接管),无需重复启动");
|
||||||
|
}
|
||||||
throw new BusinessException("任务已结束");
|
throw new BusinessException("任务已结束");
|
||||||
}
|
}
|
||||||
task.setStatus(STATUS_RUNNING);
|
|
||||||
task.setUpdatedAt(LocalDateTime.now());
|
|
||||||
fileTaskMapper.updateById(task);
|
|
||||||
taskCacheService.touchTaskHeartbeat(taskId);
|
taskCacheService.touchTaskHeartbeat(taskId);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -412,6 +424,53 @@ public class AppearancePatentTaskService {
|
|||||||
|
|
||||||
public void submitResult(Long taskId, AppearancePatentSubmitResultRequest request) {
|
public void submitResult(Long taskId, AppearancePatentSubmitResultRequest request) {
|
||||||
submitResultLocked(taskId, request);
|
submitResultLocked(taskId, request);
|
||||||
|
// 分片补传后自动恢复终态失败的组装 job —— 走与删除品牌同一套口径,
|
||||||
|
// 此前外观专利没有接该入口,分片补齐后只能人工重置 job(线上任务 28459 即如此)。
|
||||||
|
maybeRecoverTerminalFailedAssemble(taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 补传恢复:此前因分片缺失导致组装 job 重试耗尽(终态失败),客户端补传缺口后
|
||||||
|
* 把失败的组装 job 重置为 PENDING 重新派发({@code resetTerminalFailedForRecovery} 自行补发 dispatch 事件)。
|
||||||
|
*
|
||||||
|
* <p>best-effort:恢复失败不得影响补传本身——分片已经落库,恢复只是让后续组装继续推进。
|
||||||
|
* 常态(无终态失败 job)下只查两次即返回,不触发分片扫描。
|
||||||
|
*/
|
||||||
|
private void maybeRecoverTerminalFailedAssemble(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
FileResultEntity result = findResultRecord(taskId);
|
||||||
|
if (result == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (taskFileJobService.hasSuccessfulAssembleJob(taskId, MODULE_TYPE, result.getId())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!taskFileJobService.isTerminalFailedAssembleJob(taskId, MODULE_TYPE, result.getId())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!isResultSubmissionComplete(taskId)) {
|
||||||
|
log.info("[appearance-patent] 组装 job 终态失败但分片仍未补传完整,暂不恢复 taskId={} resultId={}",
|
||||||
|
taskId, result.getId());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
log.info("[appearance-patent] 分片已补传完整,恢复终态失败的组装 job taskId={} resultId={}",
|
||||||
|
taskId, result.getId());
|
||||||
|
taskFileJobService.resetTerminalFailedForRecovery(taskId, MODULE_TYPE, result.getId());
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[appearance-patent] 补传恢复检查失败(不影响本次补传)taskId={} err={}", taskId, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 按 task 取结果行(不创建);不存在返回 null。 */
|
||||||
|
private FileResultEntity findResultRecord(Long taskId) {
|
||||||
|
List<FileResultEntity> rows = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||||
|
.eq(FileResultEntity::getTaskId, taskId)
|
||||||
|
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.last("limit 1"));
|
||||||
|
return rows == null || rows.isEmpty() ? null : rows.getFirst();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void submitResultLocked(Long taskId, AppearancePatentSubmitResultRequest request) {
|
private void submitResultLocked(Long taskId, AppearancePatentSubmitResultRequest request) {
|
||||||
@@ -512,20 +571,28 @@ public class AppearancePatentTaskService {
|
|||||||
scheduleLlmPipelineForSubmittedChunk(context);
|
scheduleLlmPipelineForSubmittedChunk(context);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 删除任务。
|
||||||
|
*
|
||||||
|
* <p>事务边界:远端载荷删除与缓存清理移到事务提交后执行 —— 单次删除超时 120s、重试 3 次,
|
||||||
|
* 放在 DB 事务里会让连接被长时间占用(项目在 TaskScopePayloadStorageService 已写明该约束)。
|
||||||
|
*/
|
||||||
@Transactional
|
@Transactional
|
||||||
public void deleteTask(Long taskId, Long userId) {
|
public void deleteTask(Long taskId, Long userId) {
|
||||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !Objects.equals(userId, task.getUserId())) {
|
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !Objects.equals(userId, task.getUserId())) {
|
||||||
throw new BusinessException("任务不存在");
|
throw new BusinessException("任务不存在");
|
||||||
}
|
}
|
||||||
|
List<String> payloads = collectTransientTaskPayloads(taskId);
|
||||||
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>().eq(FileResultEntity::getTaskId, taskId).eq(FileResultEntity::getModuleType, MODULE_TYPE));
|
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>().eq(FileResultEntity::getTaskId, taskId).eq(FileResultEntity::getModuleType, MODULE_TYPE));
|
||||||
deleteTransientTaskPayloads(taskId);
|
|
||||||
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>().eq(TaskScopeStateEntity::getTaskId, taskId).eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>().eq(TaskScopeStateEntity::getTaskId, taskId).eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
||||||
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>().eq(TaskChunkEntity::getTaskId, taskId).eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>().eq(TaskChunkEntity::getTaskId, taskId).eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||||
taskCacheService.deleteTaskCache(taskId);
|
|
||||||
// 同步清理 task_file_job,避免被删除任务遗留的 PENDING/FAILED 行被 TaskResultFileJobWorker 反复扫描出 task not found。
|
// 同步清理 task_file_job,避免被删除任务遗留的 PENDING/FAILED 行被 TaskResultFileJobWorker 反复扫描出 task not found。
|
||||||
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
|
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
|
||||||
fileTaskMapper.deleteById(taskId);
|
fileTaskMapper.deleteById(taskId);
|
||||||
|
// 事务提交后再做远端删除与缓存清理
|
||||||
|
deletePayloadsAfterCommit(payloads, taskId);
|
||||||
|
runAfterCommit(() -> taskCacheService.deleteTaskCache(taskId));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void deleteHistory(Long resultId, Long userId) {
|
public void deleteHistory(Long resultId, Long userId) {
|
||||||
@@ -593,6 +660,7 @@ public class AppearancePatentTaskService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
finalizeNoUploadStaleTasks();
|
||||||
}
|
}
|
||||||
|
|
||||||
public void debugFinalizeStaleTask(Long taskId) {
|
public void debugFinalizeStaleTask(Long taskId) {
|
||||||
@@ -653,6 +721,64 @@ public class AppearancePatentTaskService {
|
|||||||
return updatedMillis <= thresholdMillis;
|
return updatedMillis <= thresholdMillis;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 二次判死:Python 心跳正常(Redis heartbeat 新鲜)但连续 N 分钟无结果分片上报。
|
||||||
|
*
|
||||||
|
* <p>既有判定以 Redis heartbeat 为 stale 主信号(P1-7),但该心跳随 Python 的 HTTP 心跳
|
||||||
|
* 每分钟刷新——主线程卡死时心跳线程照发,任务永远不 stale(与生产 28131 同型缺口)。
|
||||||
|
* 本线改看 biz_task_scope_state.last_chunk_at(仅在 persistSubmittedChunk 上传分片时刷新),
|
||||||
|
* 命中后走既有 finalizeStaleTask(封口上传 + LLM 收尾,不粗暴杀任务)。
|
||||||
|
* 从未上报(无 scope 行或全 NULL)的任务跳过,避免误杀首批较慢的正常任务。
|
||||||
|
*/
|
||||||
|
private void finalizeNoUploadStaleTasks() {
|
||||||
|
long minutes = properties.getNoResultUploadTimeoutMinutes();
|
||||||
|
if (minutes <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
List<FileTaskEntity> tasks = listStaleFinalizeCandidates();
|
||||||
|
if (tasks.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(minutes);
|
||||||
|
long heartbeatThresholdMillis = LocalDateTime.now()
|
||||||
|
.minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes()))
|
||||||
|
.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
|
||||||
|
Map<Long, LocalDateTime> lastResultAtByTaskId = new HashMap<>();
|
||||||
|
for (TaskScopeLastChunkDto dto : taskScopeStateMapper.selectLastChunkAtByTaskIds(
|
||||||
|
tasks.stream().map(FileTaskEntity::getId).toList())) {
|
||||||
|
if (dto.taskId() != null && dto.lastChunkAt() != null) {
|
||||||
|
lastResultAtByTaskId.put(dto.taskId(), dto.lastChunkAt());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (FileTaskEntity task : tasks) {
|
||||||
|
if (isHeartbeatStale(task, heartbeatThresholdMillis)) {
|
||||||
|
// 心跳已 stale:归既有心跳线处理
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
LocalDateTime lastResultAt = lastResultAtByTaskId.get(task.getId());
|
||||||
|
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(task.getId(), 0L);
|
||||||
|
if (lockHandle == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try (lockHandle) {
|
||||||
|
log.warn("[appearance-patent] 心跳正常但 {} 分钟无结果分片上报,按卡死收尾 taskId={} lastResultAt={}",
|
||||||
|
minutes, task.getId(), lastResultAt);
|
||||||
|
String error = "Python heartbeat alive but no result chunk uploaded for " + minutes + " minutes";
|
||||||
|
if (transactionManager != null) {
|
||||||
|
inNewTransaction(() -> {
|
||||||
|
finalizeStaleTask(task.getId(), error);
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
finalizeStaleTask(task.getId(), error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* /result 提交的纯计算阶段(无事务注解):读任务 + 校验归属 + 分组展开 +
|
* /result 提交的纯计算阶段(无事务注解):读任务 + 校验归属 + 分组展开 +
|
||||||
* 序列化 + payload 存储 + 哈希预计算,全部在事务开始前完成。
|
* 序列化 + payload 存储 + 哈希预计算,全部在事务开始前完成。
|
||||||
@@ -986,6 +1112,14 @@ public class AppearancePatentTaskService {
|
|||||||
long activeAssembleJobs = taskFileJobService.countActiveAssembleJobs(taskId, MODULE_TYPE);
|
long activeAssembleJobs = taskFileJobService.countActiveAssembleJobs(taskId, MODULE_TYPE);
|
||||||
log.info("[appearance-patent] stale recovery probe taskId={} uploadComplete={} activeAssembleJobs={} persistedRows={}",
|
log.info("[appearance-patent] stale recovery probe taskId={} uploadComplete={} activeAssembleJobs={} persistedRows={}",
|
||||||
taskId, uploadComplete, activeAssembleJobs, hasPersistedResultRows(taskId));
|
taskId, uploadComplete, activeAssembleJobs, hasPersistedResultRows(taskId));
|
||||||
|
// 已有「重试耗尽且已终态收尾」的 assemble job:说明恢复已经试过、缺失是永久的。
|
||||||
|
// 再重建只会每 30 秒空转一轮,而且恢复过程刷新任务心跳会让任务永远 RUNNING
|
||||||
|
// (线上任务 28459 实测:48 分钟里每隔 30 秒重建一次 job)。返回 false 交给
|
||||||
|
// finalizeStaleTask 按失败收尾,用户看到明确失败而不是无限等待。
|
||||||
|
if (taskFileJobService.hasExhaustedAssembleJob(taskId, MODULE_TYPE)) {
|
||||||
|
log.warn("[appearance-patent] stale recovery 放弃:已有重试耗尽的 assemble job,按失败收尾 taskId={}", taskId);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
if (!hasPersistedResultRows(taskId)) {
|
if (!hasPersistedResultRows(taskId)) {
|
||||||
log.info("[appearance-patent] stale recovery aborted because no persisted rows taskId={}", taskId);
|
log.info("[appearance-patent] stale recovery aborted because no persisted rows taskId={}", taskId);
|
||||||
return false;
|
return false;
|
||||||
@@ -1136,6 +1270,7 @@ public class AppearancePatentTaskService {
|
|||||||
if (rows == null || rows.isEmpty()) {
|
if (rows == null || rows.isEmpty()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
String conflictDetail = "未发生冲突";
|
||||||
for (int attempt = 1; attempt <= CHUNK_PAYLOAD_MERGE_RETRY_LIMIT; attempt++) {
|
for (int attempt = 1; attempt <= CHUNK_PAYLOAD_MERGE_RETRY_LIMIT; attempt++) {
|
||||||
TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
@@ -1177,13 +1312,35 @@ public class AppearancePatentTaskService {
|
|||||||
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
|
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
// CAS 冲突:读回行上的当前哈希,否则只有一句 conflict 无从判断是谁改的(线上 28459 至今未能定位)
|
||||||
|
String currentHash = currentPayloadHash(chunk.getId());
|
||||||
|
conflictDetail = "期望hash=" + oldPayloadHash + " 当前hash=" + currentHash;
|
||||||
if (attempt < CHUNK_PAYLOAD_MERGE_RETRY_LIMIT) {
|
if (attempt < CHUNK_PAYLOAD_MERGE_RETRY_LIMIT) {
|
||||||
log.warn("[appearance-patent] chunk payload CAS conflict, retry merge taskId={} scopeHash={} chunk={} attempt={}/{}",
|
log.warn("[appearance-patent] chunk payload CAS conflict, retry merge taskId={} scopeHash={} chunk={} attempt={}/{} 期望hash={} 当前hash={}",
|
||||||
taskId, scopeHash, chunkIndex, attempt, CHUNK_PAYLOAD_MERGE_RETRY_LIMIT);
|
taskId, scopeHash, chunkIndex, attempt, CHUNK_PAYLOAD_MERGE_RETRY_LIMIT, oldPayloadHash, currentHash);
|
||||||
|
// 还要重试:这次写的对象会被下次重写,先删掉避免堆积
|
||||||
|
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
||||||
|
} else {
|
||||||
|
// 终局失败:保留这次写入的版本化对象,作为读路径「同槽位兄弟对象」兜底的恢复源。
|
||||||
|
// 行没指过去不该让该分片永久判死——删掉它才是线上 28459 丢数据的形态。
|
||||||
|
log.error("[appearance-patent] chunk payload 合并冲突重试耗尽,保留兜底对象 taskId={} scopeHash={} chunk={} 期望hash={} 当前hash={} 保留对象={}",
|
||||||
|
taskId, scopeHash, chunkIndex, oldPayloadHash, currentHash, storedPayload);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
throw new IllegalStateException("appearance patent chunk payload update conflict");
|
throw new IllegalStateException("appearance patent chunk payload update conflict " + conflictDetail);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 读回 chunk 行当前的 payload_hash,用于 CAS 冲突定位(行已不存在/读取失败时返回可读标记)。 */
|
||||||
|
private String currentPayloadHash(Long chunkId) {
|
||||||
|
if (chunkId == null) {
|
||||||
|
return "chunkId 为空";
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
TaskChunkEntity latest = taskChunkMapper.selectById(chunkId);
|
||||||
|
return latest == null ? "行已不存在" : latest.getPayloadHash();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
return "读取失败:" + ex.getMessage();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<AppearancePatentResultRowDto> expandRows(List<AppearancePatentResultRowDto> representatives,
|
private List<AppearancePatentResultRowDto> expandRows(List<AppearancePatentResultRowDto> representatives,
|
||||||
@@ -2836,12 +2993,35 @@ public class AppearancePatentTaskService {
|
|||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
log.warn("[appearance-patent] read chunk payload failed taskId={} chunk={} err={}",
|
log.warn("[appearance-patent] read chunk payload failed taskId={} chunk={} err={}",
|
||||||
chunk.getTaskId(), chunk.getChunkIndex(), ex.getMessage());
|
chunk.getTaskId(), chunk.getChunkIndex(), ex.getMessage());
|
||||||
|
if (isPayloadMissing(ex)) {
|
||||||
|
// payload 对象已不在(被清理或从未写入):重试多少次都读不回来。继续抛会让
|
||||||
|
// ASSEMBLE_RESULT job 的终态回调每轮重跑兜底组装 → 再读同一个缺失对象 → 无限循环
|
||||||
|
// (线上任务 28459 每 10~30 秒重试一次)。跳过该分片,让任务按已有分片出部分结果,
|
||||||
|
// 与品牌/相似ASIN「失败也产出可下载的部分结果」同一口径。
|
||||||
|
log.warn("[appearance-patent] chunk payload 已不存在,跳过该分片(任务按已有分片出结果)"
|
||||||
|
+ " taskId={} chunk={}", chunk.getTaskId(), chunk.getChunkIndex());
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
throw new BusinessException("appearance patent chunk payload read failed chunk="
|
throw new BusinessException("appearance patent chunk payload read failed chunk="
|
||||||
+ chunk.getChunkIndex() + ": " + ex.getMessage(), ex);
|
+ chunk.getChunkIndex() + ": " + ex.getMessage(), ex);
|
||||||
}
|
}
|
||||||
return rows;
|
return rows;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** payload 对象已不存在(RustFS 返回 NoSuchKey:message 为 "The specified key does not exist.")。
|
||||||
|
* 只有这种"重试也没用"的缺失才允许跳过;网络类失败仍照旧抛出以便重试。 */
|
||||||
|
private static boolean isPayloadMissing(Throwable error) {
|
||||||
|
Throwable cursor = error;
|
||||||
|
while (cursor != null) {
|
||||||
|
String message = cursor.getMessage();
|
||||||
|
if (message != null && message.contains("does not exist")) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
cursor = cursor.getCause();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
private String rowKey(AppearancePatentParsedRowVo row) {
|
private String rowKey(AppearancePatentParsedRowVo row) {
|
||||||
if (row == null) {
|
if (row == null) {
|
||||||
return "";
|
return "";
|
||||||
@@ -2923,18 +3103,27 @@ public class AppearancePatentTaskService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void deleteTransientTaskPayloads(Long taskId) {
|
/** 只读收集任务范围/分片载荷指针,供事务提交后做远端删除。 */
|
||||||
|
private List<String> collectTransientTaskPayloads(Long taskId) {
|
||||||
if (taskId == null || taskId <= 0) {
|
if (taskId == null || taskId <= 0) {
|
||||||
return;
|
return List.of();
|
||||||
}
|
}
|
||||||
|
List<String> payloads = new ArrayList<>();
|
||||||
List<TaskScopeStateEntity> scopes = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
List<TaskScopeStateEntity> scopes = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
|
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
|
||||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
||||||
if (scopes != null) {
|
if (scopes != null) {
|
||||||
for (TaskScopeStateEntity scope : scopes) {
|
for (TaskScopeStateEntity scope : scopes) {
|
||||||
transientPayloadStorageService.deletePayloadIfPresent(scope.getParsedPayloadJson());
|
if (scope == null) {
|
||||||
transientPayloadStorageService.deletePayloadIfPresent(scope.getStateJson());
|
continue;
|
||||||
|
}
|
||||||
|
if (scope.getParsedPayloadJson() != null && !scope.getParsedPayloadJson().isBlank()) {
|
||||||
|
payloads.add(scope.getParsedPayloadJson());
|
||||||
|
}
|
||||||
|
if (scope.getStateJson() != null && !scope.getStateJson().isBlank()) {
|
||||||
|
payloads.add(scope.getStateJson());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
@@ -2943,9 +3132,49 @@ public class AppearancePatentTaskService {
|
|||||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||||
if (chunks != null) {
|
if (chunks != null) {
|
||||||
for (TaskChunkEntity chunk : chunks) {
|
for (TaskChunkEntity chunk : chunks) {
|
||||||
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
|
if (chunk != null && chunk.getPayloadJson() != null && !chunk.getPayloadJson().isBlank()) {
|
||||||
|
payloads.add(chunk.getPayloadJson());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return payloads;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 事务提交后逐条删远端载荷;无事务时立即执行(与 TaskResultItemService 同款兜底)。 */
|
||||||
|
private void deletePayloadsAfterCommit(List<String> payloads, Long taskId) {
|
||||||
|
if (payloads == null || payloads.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
runAfterCommit(() -> {
|
||||||
|
for (String payload : payloads) {
|
||||||
|
try {
|
||||||
|
transientPayloadStorageService.deletePayloadIfPresent(payload);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[appearance-patent] 载荷删除失败 taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 删除范围/分片的远端载荷(保留给无事务的清理链路调用)。
|
||||||
|
*/
|
||||||
|
private void deleteTransientTaskPayloads(Long taskId) {
|
||||||
|
deletePayloadsAfterCommit(collectTransientTaskPayloads(taskId), taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 有活动事务则注册 afterCommit,否则立即执行。 */
|
||||||
|
private void runAfterCommit(Runnable action) {
|
||||||
|
if (TransactionSynchronizationManager.isSynchronizationActive()) {
|
||||||
|
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
|
||||||
|
@Override
|
||||||
|
public void afterCommit() {
|
||||||
|
action.run();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
action.run();
|
||||||
}
|
}
|
||||||
|
|
||||||
private record SubmitContext(FileTaskEntity task,
|
private record SubmitContext(FileTaskEntity task,
|
||||||
|
|||||||
+34
-10
@@ -95,10 +95,14 @@ public class BrandCheckClient {
|
|||||||
|
|
||||||
public BrandCheckBatchResult checkAll(List<String> brands, String strategy) {
|
public BrandCheckBatchResult checkAll(List<String> brands, String strategy) {
|
||||||
List<String> distinctBrands = distinctNonBlank(brands);
|
List<String> distinctBrands = distinctNonBlank(brands);
|
||||||
|
// 整批共用一个耗时预算:上游 16890 卡死时,单品牌 10 次重试曾把一次分片回传拖到
|
||||||
|
// 103.5 秒(taskId 28599),客户端重试预算耗尽后中止了整个采集。预算用尽即停止重试。
|
||||||
|
long budgetMillis = properties.getTotalTimeoutMillis();
|
||||||
|
long deadlineNanos = budgetMillis > 0L ? System.nanoTime() + budgetMillis * 1_000_000L : Long.MAX_VALUE;
|
||||||
List<CompletableFuture<BrandCheckOutcome>> futures = new ArrayList<>(distinctBrands.size());
|
List<CompletableFuture<BrandCheckOutcome>> futures = new ArrayList<>(distinctBrands.size());
|
||||||
for (String brand : distinctBrands) {
|
for (String brand : distinctBrands) {
|
||||||
futures.add(CompletableFuture.supplyAsync(
|
futures.add(CompletableFuture.supplyAsync(
|
||||||
() -> checkOneBrand(brand, strategy), checkExecutor));
|
() -> checkOneBrand(brand, strategy, deadlineNanos), checkExecutor));
|
||||||
}
|
}
|
||||||
List<Object> failedData = new ArrayList<>();
|
List<Object> failedData = new ArrayList<>();
|
||||||
List<Object> queryFailedData = new ArrayList<>();
|
List<Object> queryFailedData = new ArrayList<>();
|
||||||
@@ -110,20 +114,29 @@ public class BrandCheckClient {
|
|||||||
return new BrandCheckBatchResult(distinctBrands, failedData, queryFailedData);
|
return new BrandCheckBatchResult(distinctBrands, failedData, queryFailedData);
|
||||||
}
|
}
|
||||||
|
|
||||||
private BrandCheckOutcome checkOneBrand(String brand, String strategy) {
|
private BrandCheckOutcome checkOneBrand(String brand, String strategy, long deadlineNanos) {
|
||||||
int attempts = Math.max(1, properties.getRetryTimes());
|
int attempts = Math.max(1, properties.getRetryTimes());
|
||||||
BrandCheckResponse response = null;
|
BrandCheckResponse response = null;
|
||||||
Exception lastFailure = null;
|
Exception lastFailure = null;
|
||||||
for (int attempt = 1; attempt <= attempts; attempt++) {
|
for (int attempt = 1; attempt <= attempts; attempt++) {
|
||||||
|
// 预算用尽就不再重试,按查询失败收尾。只掐「重试」不打断已发出的请求,
|
||||||
|
// 故最坏耗时 ≈ 预算 + 一次请求的读超时;首轮始终执行,避免上游只是慢一点时被误降级。
|
||||||
|
if (attempt > 1 && System.nanoTime() >= deadlineNanos) {
|
||||||
|
log.warn("[brand-check] 整批耗时预算用尽,停止重试 brand={} attempt={}/{} lastErr={}",
|
||||||
|
brand, attempt, attempts,
|
||||||
|
lastFailure == null ? "query_faild_data 持续非空" : lastFailure.getMessage());
|
||||||
|
break;
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
response = check(brand, strategy);
|
response = check(brand, strategy);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
lastFailure = ex;
|
lastFailure = ex;
|
||||||
response = null;
|
response = null;
|
||||||
if (attempt < attempts) {
|
if (attempt < attempts) {
|
||||||
log.warn("[brand-check] 请求异常将重试 brand={} attempt={}/{} err={}",
|
long retryDelayMillis = retryDelayMillis(attempt);
|
||||||
brand, attempt, attempts, ex.getMessage());
|
log.warn("[brand-check] 请求异常将重试 brand={} attempt={}/{} 等待={}ms err={}",
|
||||||
sleepBeforeRetry();
|
brand, attempt, attempts, retryDelayMillis, ex.getMessage());
|
||||||
|
sleepBeforeRetry(retryDelayMillis);
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -132,9 +145,10 @@ public class BrandCheckClient {
|
|||||||
return new BrandCheckOutcome(nullToEmpty(response.getFaildData()), List.of());
|
return new BrandCheckOutcome(nullToEmpty(response.getFaildData()), List.of());
|
||||||
}
|
}
|
||||||
if (attempt < attempts) {
|
if (attempt < attempts) {
|
||||||
log.warn("[brand-check] 服务端返回查询失败将重试 brand={} attempt={}/{}",
|
long retryDelayMillis = retryDelayMillis(attempt);
|
||||||
brand, attempt, attempts);
|
log.warn("[brand-check] 服务端返回查询失败将重试 brand={} attempt={}/{} 等待={}ms",
|
||||||
sleepBeforeRetry();
|
brand, attempt, attempts, retryDelayMillis);
|
||||||
|
sleepBeforeRetry(retryDelayMillis);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
log.warn("[brand-check] 重试耗尽判定查询失败 brand={} attempts={} lastErr={}",
|
log.warn("[brand-check] 重试耗尽判定查询失败 brand={} attempts={} lastErr={}",
|
||||||
@@ -144,8 +158,18 @@ public class BrandCheckClient {
|
|||||||
response == null ? List.of(brand) : nullToEmpty(response.getQueryFaildData()));
|
response == null ? List.of(brand) : nullToEmpty(response.getQueryFaildData()));
|
||||||
}
|
}
|
||||||
|
|
||||||
private void sleepBeforeRetry() {
|
/**
|
||||||
long delayMillis = Math.max(0L, properties.getRetryIntervalMillis());
|
* 第 attempt 次重试前的等待毫秒数:按基准间隔随轮次递增后封顶。
|
||||||
|
* 限流窗口通常只有几秒,固定 1s 间隔反复打过去救不回来;递增等待能覆盖窗口,
|
||||||
|
* 封顶则保证单个品牌不会长时间占住查询线程(并发度只有 3)。
|
||||||
|
*/
|
||||||
|
private long retryDelayMillis(int attempt) {
|
||||||
|
long base = Math.max(0L, properties.getRetryIntervalMillis());
|
||||||
|
long cap = Math.max(base, properties.getRetryMaxIntervalMillis());
|
||||||
|
return Math.min(base * Math.max(1, attempt), cap);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void sleepBeforeRetry(long delayMillis) {
|
||||||
if (delayMillis <= 0L) {
|
if (delayMillis <= 0L) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|||||||
+56
@@ -0,0 +1,56 @@
|
|||||||
|
package com.nanri.aiimage.modules.brand.controller;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.api.ApiResponse;
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||||
|
import com.nanri.aiimage.modules.brand.model.dto.BrandTaskAbortRequest;
|
||||||
|
import com.nanri.aiimage.modules.brand.service.BrandTaskService;
|
||||||
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
|
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||||
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.web.bind.annotation.PathVariable;
|
||||||
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestBody;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 内部接口:品牌任务中止上报,供主机 A 品牌检测服务(15126)在爬取不可继续时调用
|
||||||
|
* (如 WIPO 连续限流熔断)。Java 侧用已收到的结果分片部分组装结果文件并落 failed +
|
||||||
|
* 真实原因,避免任务悬挂到心跳超时被判「前端长时间无响应」、已跑出的数据无法下载。
|
||||||
|
*
|
||||||
|
* <p>鉴权:仅凭 X-Internal-Token(与容器 AIIMAGE_INTERNAL_TOKEN / 宿主机
|
||||||
|
* ~/.aiimage/internal-token 同值)。/api/internal 前缀虽在 AdminApiGuardFilter 兜底
|
||||||
|
* 名单内、可信令牌会放行,controller 仍须自校验——防止配置漂移时匿名可达。
|
||||||
|
*/
|
||||||
|
@RestController
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
@RequestMapping("/api/internal/brand")
|
||||||
|
@Tag(name = "内部接口", description = "服务间调用(X-Internal-Token 鉴权)。")
|
||||||
|
public class InternalBrandTaskController {
|
||||||
|
|
||||||
|
private final BrandTaskService brandTaskService;
|
||||||
|
private final AdminAuthSupport adminAuthSupport;
|
||||||
|
|
||||||
|
@PostMapping("/tasks/{taskId}/abort")
|
||||||
|
@Operation(summary = "上报品牌任务中止(爬取方调用)",
|
||||||
|
description = "用已收到的结果分片部分组装结果文件(未检测品牌单独成 sheet)并落 failed + 真实原因;幂等,终态任务直接返回。")
|
||||||
|
public ApiResponse<Map<String, Object>> abortTask(HttpServletRequest request,
|
||||||
|
@PathVariable Long taskId,
|
||||||
|
@RequestBody(required = false) BrandTaskAbortRequest body) {
|
||||||
|
if (!adminAuthSupport.isTrustedInternalToken(request)) {
|
||||||
|
log.warn("[internal-brand-abort] 拒绝未携带可信内部令牌的请求 taskId={} remoteAddr={}",
|
||||||
|
taskId, request.getRemoteAddr());
|
||||||
|
throw new BusinessException(401, "未授权");
|
||||||
|
}
|
||||||
|
String errorMessage = body == null ? null : body.getErrorMessage();
|
||||||
|
log.info("[internal-brand-abort] 收到中止上报 taskId={} remoteAddr={} msg={}",
|
||||||
|
taskId, request.getRemoteAddr(), errorMessage);
|
||||||
|
return ApiResponse.success(brandTaskService.abortTask(taskId, errorMessage));
|
||||||
|
}
|
||||||
|
}
|
||||||
+24
@@ -3,7 +3,31 @@ package com.nanri.aiimage.modules.brand.mapper;
|
|||||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||||
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
|
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
|
||||||
import org.apache.ibatis.annotations.Mapper;
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
|
import org.apache.ibatis.annotations.Param;
|
||||||
|
import org.apache.ibatis.annotations.Select;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
@Mapper
|
@Mapper
|
||||||
public interface BrandCrawlTaskMapper extends BaseMapper<BrandCrawlTaskEntity> {
|
public interface BrandCrawlTaskMapper extends BaseMapper<BrandCrawlTaskEntity> {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 查询超过保留期的终态品牌检测任务 id(保留期清理用,只取 id 不拉整行——历史行的
|
||||||
|
* file_paths/result_paths JSON 字段可能很大)。
|
||||||
|
*
|
||||||
|
* <p>终态集合与 {@code BrandTaskService} 的状态机一致(success/failed/cancelled),
|
||||||
|
* pending/running 绝不返回(删了正在跑的任务,结果回传会找不到任务行)。
|
||||||
|
* 时间线用 updated_at,与 BrandTaskStaleRepairSpiImpl 的陈旧判定同款口径,
|
||||||
|
* 可命中 V120 的 idx_brand_crawl_task_status_updated(status, updated_at) 索引。
|
||||||
|
*/
|
||||||
|
@Select("""
|
||||||
|
SELECT id FROM brand_crawl_tasks
|
||||||
|
WHERE status IN ('success', 'failed', 'cancelled')
|
||||||
|
AND updated_at < #{cutoff}
|
||||||
|
ORDER BY id ASC
|
||||||
|
LIMIT #{batchSize}
|
||||||
|
""")
|
||||||
|
List<Long> selectExpiredTerminalTaskIds(@Param("cutoff") LocalDateTime cutoff,
|
||||||
|
@Param("batchSize") int batchSize);
|
||||||
}
|
}
|
||||||
|
|||||||
+3
@@ -18,4 +18,7 @@ public class BrandFileAggregateCacheDto {
|
|||||||
private Boolean completed = false;
|
private Boolean completed = false;
|
||||||
private List<BrandInvalidBrandDto> invalidBrands = new ArrayList<>();
|
private List<BrandInvalidBrandDto> invalidBrands = new ArrayList<>();
|
||||||
private List<String> queryFailedBrands = new ArrayList<>();
|
private List<String> queryFailedBrands = new ArrayList<>();
|
||||||
|
/** 已判定保留的品牌:与 invalidBrands / queryFailedBrands 一起构成「已检测品牌」,
|
||||||
|
* 失败任务的未检测品牌 = 源文件品牌 - 三者并集(部分组装时写「未检测品牌」sheet)。 */
|
||||||
|
private List<String> keptBrands = new ArrayList<>();
|
||||||
}
|
}
|
||||||
|
|||||||
+13
@@ -0,0 +1,13 @@
|
|||||||
|
package com.nanri.aiimage.modules.brand.model.dto;
|
||||||
|
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Schema(description = "品牌任务中止上报请求(内部接口,由爬取方 15126 调用)。")
|
||||||
|
public class BrandTaskAbortRequest {
|
||||||
|
|
||||||
|
@Schema(description = "中止原因,会原样写入任务 error_message,前端任务列表展示该文案。",
|
||||||
|
example = "连续 8 次请求被 WIPO 限流(返回 Forbidden),已中止任务;请检查代理配置或错峰重跑")
|
||||||
|
private String errorMessage;
|
||||||
|
}
|
||||||
+2
@@ -60,6 +60,8 @@ public class BrandTaskProgressCacheService {
|
|||||||
values.put("finished_files", String.valueOf(Math.max(finishedFiles, 0)));
|
values.put("finished_files", String.valueOf(Math.max(finishedFiles, 0)));
|
||||||
values.put("updated_at", now);
|
values.put("updated_at", now);
|
||||||
values.put("last_heartbeat_at", now);
|
values.put("last_heartbeat_at", now);
|
||||||
|
// 结果上报专属信号(二次判死线用):心跳/touchHeartbeat 不写它,只有结果回传才刷新
|
||||||
|
values.put("last_result_at", now);
|
||||||
try {
|
try {
|
||||||
stringRedisTemplate.opsForHash().putAll(key, values);
|
stringRedisTemplate.opsForHash().putAll(key, values);
|
||||||
stringRedisTemplate.expire(key, ttl());
|
stringRedisTemplate.expire(key, ttl());
|
||||||
|
|||||||
+93
@@ -0,0 +1,93 @@
|
|||||||
|
package com.nanri.aiimage.modules.brand.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
|
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.scheduling.annotation.Scheduled;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 品牌检测任务(brand_crawl_tasks)的保留期清理(2026-09 审核:该表只增不删,永久累积)。
|
||||||
|
*
|
||||||
|
* <p>该表自建、不写 biz_file_task,故不在 ModuleHistoryCleanupService 的清理名单里,
|
||||||
|
* 此前没有任何删除路径。这里只查过期终态任务的 id,逐个走
|
||||||
|
* {@link BrandTaskService#deleteTask(Long)} 既有删除入口 —— 它已处理任务行删除 +
|
||||||
|
* 存储数据清理(brandTaskStorageService.deleteTaskData)+ 进度缓存清理,本类不重新实现删除逻辑。
|
||||||
|
*
|
||||||
|
* <p>双节点用 job 锁保证单实例执行;每批小批量(默认 50)逐个删,避免单次跑太久占住锁。
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class BrandTaskRetentionService {
|
||||||
|
|
||||||
|
/** 单轮最多处理的批次数(每批 batchSize 个任务),剩余留给下一轮。 */
|
||||||
|
static final int MAX_BATCHES_PER_RUN = 20;
|
||||||
|
|
||||||
|
private final BrandCrawlTaskMapper brandCrawlTaskMapper;
|
||||||
|
private final BrandTaskService brandTaskService;
|
||||||
|
private final DistributedJobLockService distributedJobLockService;
|
||||||
|
|
||||||
|
@Value("${aiimage.brand.task-retention-days:90}")
|
||||||
|
private int retentionDays = 90;
|
||||||
|
|
||||||
|
@Value("${aiimage.brand.task-retention-batch-size:50}")
|
||||||
|
private int retentionBatchSize = 50;
|
||||||
|
|
||||||
|
@Scheduled(cron = "${aiimage.brand.task-retention-cron:0 45 4 * * *}")
|
||||||
|
public void purgeExpiredTasks() {
|
||||||
|
int days = Math.max(1, retentionDays);
|
||||||
|
int batchSize = Math.max(1, retentionBatchSize);
|
||||||
|
LocalDateTime cutoff = LocalDateTime.now().minusDays(days);
|
||||||
|
|
||||||
|
// 单轮 20 批 × 50 个任务的删除(含存储数据清理)可能跑较久,锁 TTL 给足 30 分钟
|
||||||
|
DistributedJobLockService.LockHandle lockHandle =
|
||||||
|
distributedJobLockService.tryLock("brand:task-retention", Duration.ofMinutes(30));
|
||||||
|
if (lockHandle == null) {
|
||||||
|
log.info("[brand-retention] 任务保留清理跳过:另一实例持锁");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try (lockHandle) {
|
||||||
|
int totalDeleted = 0;
|
||||||
|
int totalFailed = 0;
|
||||||
|
int batches = 0;
|
||||||
|
while (batches < MAX_BATCHES_PER_RUN) {
|
||||||
|
// 只取 id:历史行的 file_paths/result_paths JSON 字段可能很大
|
||||||
|
List<Long> taskIds = brandCrawlTaskMapper.selectExpiredTerminalTaskIds(cutoff, batchSize);
|
||||||
|
if (taskIds.isEmpty()) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
batches++;
|
||||||
|
int deletedInBatch = 0;
|
||||||
|
for (Long taskId : taskIds) {
|
||||||
|
try {
|
||||||
|
brandTaskService.deleteTask(taskId);
|
||||||
|
deletedInBatch++;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
// 单个任务删除失败只记日志继续:一个坏任务不能卡住整轮
|
||||||
|
log.warn("[brand-retention] 任务删除失败 taskId={} msg={}", taskId, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
totalDeleted += deletedInBatch;
|
||||||
|
totalFailed += taskIds.size() - deletedInBatch;
|
||||||
|
if (deletedInBatch == 0) {
|
||||||
|
log.warn("[brand-retention] 整批任务均删除失败,提前结束本轮以免反复重试同一批");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (taskIds.size() < batchSize) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.info("[brand-retention] 任务保留清理完成 cutoff={} retentionDays={} deleted={} failed={} batches={}",
|
||||||
|
cutoff, days, totalDeleted, totalFailed, batches);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[brand-retention] 任务保留清理失败 cutoff={} msg={}", cutoff, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+216
-17
@@ -686,23 +686,8 @@ public class BrandTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
brandTaskProgressCacheService.updatePhase(taskId, BrandTaskProgressCacheService.PHASE_ASSEMBLING, totalCount, totalCount);
|
brandTaskProgressCacheService.updatePhase(taskId, BrandTaskProgressCacheService.PHASE_ASSEMBLING, totalCount, totalCount);
|
||||||
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "brand-result", String.valueOf(taskId)));
|
|
||||||
List<OutputEntry> outputEntries = new ArrayList<>();
|
|
||||||
try {
|
try {
|
||||||
for (BrandSourceFileDto sourceFile : sourceFiles) {
|
Map<String, Object> resultPaths = assembleAndUploadResult(taskId, strategy, sourceFiles, cachedByUrl, aggregates, false);
|
||||||
BrandFileAggregateCacheDto aggregate = aggregates.get(sourceFile.getFileUrl());
|
|
||||||
BrandParsedFileCacheDto cachedFile = cachedByUrl.get(sourceFile.getFileUrl());
|
|
||||||
if (cachedFile == null) {
|
|
||||||
throw new BusinessException("缺少原始缓存数据: " + sourceFile.getFileUrl());
|
|
||||||
}
|
|
||||||
File sourceLocalFile = resolveSourceFile(sourceFile);
|
|
||||||
String originalFilename = resolveOriginalFilename(sourceFile, sourceLocalFile);
|
|
||||||
File outputFile = buildNamedOutputFile(outputDir, buildResultFilename(originalFilename));
|
|
||||||
writeBrandWorkbook(outputFile, strategy, cachedFile, aggregate);
|
|
||||||
outputEntries.add(new OutputEntry(sourceLocalFile, originalFilename, outputFile, originalFilename));
|
|
||||||
}
|
|
||||||
brandTaskProgressCacheService.updatePhase(taskId, BrandTaskProgressCacheService.PHASE_UPLOADING, totalCount, totalCount);
|
|
||||||
Map<String, Object> resultPaths = buildAndUploadResult(taskId, outputEntries);
|
|
||||||
int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||||
.eq(BrandCrawlTaskEntity::getId, taskId)
|
.eq(BrandCrawlTaskEntity::getId, taskId)
|
||||||
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
|
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
|
||||||
@@ -736,11 +721,150 @@ public class BrandTaskService {
|
|||||||
throw businessException;
|
throw businessException;
|
||||||
}
|
}
|
||||||
throw new BusinessException(ex.getMessage());
|
throw new BusinessException(ex.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 组装并上传结果文件。partial=false 为成功收尾(要求全部文件分片收齐,由调用方校验);
|
||||||
|
* partial=true 为失败中止的部分组装:用已收到的分片出结果,未检测品牌单独成 sheet。
|
||||||
|
*/
|
||||||
|
private Map<String, Object> assembleAndUploadResult(Long taskId,
|
||||||
|
String strategy,
|
||||||
|
List<BrandSourceFileDto> sourceFiles,
|
||||||
|
Map<String, BrandParsedFileCacheDto> cachedByUrl,
|
||||||
|
Map<String, BrandFileAggregateCacheDto> aggregates,
|
||||||
|
boolean partial) throws IOException {
|
||||||
|
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "brand-result", String.valueOf(taskId)));
|
||||||
|
List<OutputEntry> outputEntries = new ArrayList<>();
|
||||||
|
try {
|
||||||
|
for (BrandSourceFileDto sourceFile : sourceFiles) {
|
||||||
|
BrandParsedFileCacheDto cachedFile = cachedByUrl.get(sourceFile.getFileUrl());
|
||||||
|
if (cachedFile == null) {
|
||||||
|
if (!partial) {
|
||||||
|
throw new BusinessException("缺少原始缓存数据: " + sourceFile.getFileUrl());
|
||||||
|
}
|
||||||
|
log.warn("[brand-assemble] taskId={} 原始缓存数据缺失,跳过该文件 fileUrl={}", taskId, sourceFile.getFileUrl());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
BrandFileAggregateCacheDto aggregate = aggregates.get(sourceFile.getFileUrl());
|
||||||
|
if (aggregate == null) {
|
||||||
|
if (!partial) {
|
||||||
|
throw new BusinessException("缺少结果聚合数据: " + sourceFile.getFileUrl());
|
||||||
|
}
|
||||||
|
aggregate = new BrandFileAggregateCacheDto();
|
||||||
|
aggregate.setFileUrl(sourceFile.getFileUrl());
|
||||||
|
}
|
||||||
|
List<String> undetectedBrands = partial ? resolveUndetectedBrands(cachedFile, aggregate) : List.of();
|
||||||
|
File sourceLocalFile = resolveSourceFile(sourceFile);
|
||||||
|
String originalFilename = resolveOriginalFilename(sourceFile, sourceLocalFile);
|
||||||
|
File outputFile = buildNamedOutputFile(outputDir, buildResultFilename(originalFilename));
|
||||||
|
writeBrandWorkbook(outputFile, strategy, cachedFile, aggregate, undetectedBrands);
|
||||||
|
outputEntries.add(new OutputEntry(sourceLocalFile, originalFilename, outputFile, originalFilename));
|
||||||
|
}
|
||||||
|
if (outputEntries.isEmpty()) {
|
||||||
|
throw new BusinessException("没有可组装的结果文件");
|
||||||
|
}
|
||||||
|
brandTaskProgressCacheService.updatePhase(taskId, BrandTaskProgressCacheService.PHASE_UPLOADING,
|
||||||
|
sourceFiles.size(), sourceFiles.size());
|
||||||
|
return buildAndUploadResult(taskId, outputEntries);
|
||||||
} finally {
|
} finally {
|
||||||
cleanupBrandResultTempFiles(outputEntries, outputDir);
|
cleanupBrandResultTempFiles(outputEntries, outputDir);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 未检测品牌 = 源文件品牌 -(保留 ∪ 不符合品牌 ∪ 查询失败品牌);按源文件出现顺序去重。 */
|
||||||
|
private List<String> resolveUndetectedBrands(BrandParsedFileCacheDto cachedFile, BrandFileAggregateCacheDto aggregate) {
|
||||||
|
List<Map<String, Object>> sourceRows = cachedFile.getRows() == null ? List.of() : cachedFile.getRows();
|
||||||
|
Set<String> handled = new LinkedHashSet<>();
|
||||||
|
handled.addAll(normalizeBrandSet(aggregate.getKeptBrands()));
|
||||||
|
handled.addAll(normalizeBrandSetFromInvalids(aggregate.getInvalidBrands()));
|
||||||
|
handled.addAll(normalizeBrandSet(aggregate.getQueryFailedBrands()));
|
||||||
|
LinkedHashSet<String> undetected = new LinkedHashSet<>();
|
||||||
|
for (Map<String, Object> rowData : sourceRows) {
|
||||||
|
String brand = normalizeCellText(Objects.toString(rowData.getOrDefault("品牌", ""), ""));
|
||||||
|
if (!brand.isBlank() && !handled.contains(brand)) {
|
||||||
|
undetected.add(brand);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return new ArrayList<>(undetected);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 爬取方(15126)中止上报:任务不可能再收到剩余分片时调用(如 WIPO 连续限流熔断)。
|
||||||
|
* 用已收到的分片部分组装结果文件并落 failed + 真实原因——否则 Java 侧任务会悬挂到
|
||||||
|
* 心跳超时被判「前端长时间无响应」,且已跑出的数据因没有 result_paths 无法下载。
|
||||||
|
* 幂等:任务已是终态时直接返回,不覆盖既有结果。
|
||||||
|
*/
|
||||||
|
public Map<String, Object> abortTask(Long taskId, String errorMessage) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
throw new BusinessException("taskId invalid");
|
||||||
|
}
|
||||||
|
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId, RESULT_SUBMIT_WAIT_MILLIS)) {
|
||||||
|
BrandCrawlTaskEntity task = requireTask(taskId);
|
||||||
|
String status = blankToDefault(task.getStatus(), STATUS_PENDING);
|
||||||
|
Map<String, Object> data = new LinkedHashMap<>();
|
||||||
|
data.put("taskId", taskId);
|
||||||
|
boolean hasResult = task.getResultPaths() != null && !task.getResultPaths().isBlank();
|
||||||
|
// success/cancelled 不动;failed 已有结果也不重复组装。failed 且无结果
|
||||||
|
// (如被心跳超时兜底判失败的历史任务)允许补组装——存量补救路径。
|
||||||
|
if (STATUS_SUCCESS.equalsIgnoreCase(status) || STATUS_CANCELLED.equalsIgnoreCase(status)
|
||||||
|
|| (STATUS_FAILED.equalsIgnoreCase(status) && hasResult)) {
|
||||||
|
log.info("[brand-abort] taskId={} 已是终态且无需补组装 status={} hasResult={},跳过",
|
||||||
|
taskId, status, hasResult);
|
||||||
|
data.put("status", status);
|
||||||
|
data.put("resultGenerated", false);
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
String message = blankToDefault(errorMessage,
|
||||||
|
blankToDefault(task.getErrorMessage(), "品牌检测任务已中止"));
|
||||||
|
List<BrandSourceFileDto> sourceFiles = parseSourceFiles(task.getFilePaths());
|
||||||
|
int totalCount = sourceFiles.size();
|
||||||
|
Map<String, BrandParsedFileCacheDto> cachedByUrl =
|
||||||
|
indexCachedFiles(brandTaskStorageService.getParsedPayload(taskId));
|
||||||
|
// 先从分片重建聚合再读取:缓存的 state_json 可能是旧版本(缺后加字段,
|
||||||
|
// 如 keptBrands)或与已落库分片不一致,直接读会把已检测品牌误判为未检测
|
||||||
|
for (BrandSourceFileDto sourceFile : sourceFiles) {
|
||||||
|
brandTaskStorageService.refreshFileAggregate(taskId, sourceFile.getFileUrl());
|
||||||
|
}
|
||||||
|
Map<String, BrandFileAggregateCacheDto> aggregates = brandTaskStorageService.getAllFileAggregates(taskId);
|
||||||
|
boolean hasChunk = aggregates.values().stream()
|
||||||
|
.anyMatch(item -> item != null && defaultInteger(item.getReceivedChunkCount()) > 0);
|
||||||
|
Map<String, Object> resultPaths = null;
|
||||||
|
if (hasChunk && !cachedByUrl.isEmpty() && !sourceFiles.isEmpty()) {
|
||||||
|
try {
|
||||||
|
resultPaths = assembleAndUploadResult(taskId, normalizeStrategy(task.getStrategy()),
|
||||||
|
sourceFiles, cachedByUrl, aggregates, true);
|
||||||
|
log.info("[brand-abort] taskId={} 部分结果组装完成 files={}", taskId, sourceFiles.size());
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[brand-abort] taskId={} 部分结果组装失败(仅标记失败) msg={}", taskId, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.info("[brand-abort] taskId={} 无已收到分片,跳过结果组装 receivedAggregates={}", taskId, aggregates.size());
|
||||||
|
}
|
||||||
|
int finishedCount = brandTaskStorageService.countCompletedFiles(taskId);
|
||||||
|
LambdaUpdateWrapper<BrandCrawlTaskEntity> wrapper = new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||||
|
.eq(BrandCrawlTaskEntity::getId, taskId)
|
||||||
|
.in(BrandCrawlTaskEntity::getStatus, STATUS_PENDING, STATUS_RUNNING, STATUS_FAILED)
|
||||||
|
.set(BrandCrawlTaskEntity::getStatus, STATUS_FAILED)
|
||||||
|
.set(BrandCrawlTaskEntity::getErrorMessage, message)
|
||||||
|
.set(BrandCrawlTaskEntity::getProgressCurrent, finishedCount)
|
||||||
|
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount);
|
||||||
|
if (resultPaths != null) {
|
||||||
|
wrapper.set(BrandCrawlTaskEntity::getResultPaths, JSONUtil.toJsonStr(resultPaths));
|
||||||
|
}
|
||||||
|
int updated = brandCrawlTaskMapper.update(null, wrapper);
|
||||||
|
if (updated > 0) {
|
||||||
|
brandTaskProgressCacheService.markFailed(taskId, message);
|
||||||
|
saveBrandProgressSnapshot(taskId, STATUS_FAILED, totalCount, finishedCount, 1, message);
|
||||||
|
}
|
||||||
|
log.info("[brand-abort] taskId={} aborted updated={} resultGenerated={} finishedFiles={}/{} msg={}",
|
||||||
|
taskId, updated, resultPaths != null, finishedCount, totalCount, message);
|
||||||
|
data.put("status", STATUS_FAILED);
|
||||||
|
data.put("resultGenerated", resultPaths != null);
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private void ensureNoDuplicateResultFiles(List<BrandCrawlResultFileDto> resultFiles) {
|
private void ensureNoDuplicateResultFiles(List<BrandCrawlResultFileDto> resultFiles) {
|
||||||
Set<String> seen = new LinkedHashSet<>();
|
Set<String> seen = new LinkedHashSet<>();
|
||||||
for (BrandCrawlResultFileDto resultFile : resultFiles) {
|
for (BrandCrawlResultFileDto resultFile : resultFiles) {
|
||||||
@@ -1034,7 +1158,8 @@ public class BrandTaskService {
|
|||||||
private void writeBrandWorkbook(File outputFile,
|
private void writeBrandWorkbook(File outputFile,
|
||||||
String strategy,
|
String strategy,
|
||||||
BrandParsedFileCacheDto cachedFile,
|
BrandParsedFileCacheDto cachedFile,
|
||||||
BrandFileAggregateCacheDto resultFile) throws IOException {
|
BrandFileAggregateCacheDto resultFile,
|
||||||
|
List<String> undetectedBrands) throws IOException {
|
||||||
String actualStrategy = normalizeStrategy(strategy);
|
String actualStrategy = normalizeStrategy(strategy);
|
||||||
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
|
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
|
||||||
workbook.setCompressTempFiles(true);
|
workbook.setCompressTempFiles(true);
|
||||||
@@ -1048,6 +1173,10 @@ public class BrandTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Set<String> invalidBrandSet = normalizeBrandSetFromInvalids(resultFile.getInvalidBrands());
|
Set<String> invalidBrandSet = normalizeBrandSetFromInvalids(resultFile.getInvalidBrands());
|
||||||
|
// 未检测品牌(任务中止、分片没到齐):主 sheet 剔除这些行、整体挪到独立 sheet,
|
||||||
|
// 避免用户把「没查过」的行误当成「已通过检测」上架
|
||||||
|
Set<String> undetectedBrandSet = normalizeBrandSet(undetectedBrands);
|
||||||
|
List<Map<String, Object>> undetectedRows = new ArrayList<>();
|
||||||
List<Map<String, Object>> sourceRows = cachedFile.getRows() == null ? List.of() : cachedFile.getRows();
|
List<Map<String, Object>> sourceRows = cachedFile.getRows() == null ? List.of() : cachedFile.getRows();
|
||||||
int writeRowIndex = 1;
|
int writeRowIndex = 1;
|
||||||
for (Map<String, Object> rowData : sourceRows) {
|
for (Map<String, Object> rowData : sourceRows) {
|
||||||
@@ -1055,6 +1184,10 @@ public class BrandTaskService {
|
|||||||
if (!brand.isBlank() && invalidBrandSet.contains(brand)) {
|
if (!brand.isBlank() && invalidBrandSet.contains(brand)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if (!brand.isBlank() && undetectedBrandSet.contains(brand)) {
|
||||||
|
undetectedRows.add(rowData);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
var row = mainSheet.createRow(writeRowIndex++);
|
var row = mainSheet.createRow(writeRowIndex++);
|
||||||
for (int colIndex = 0; colIndex < columns.size(); colIndex++) {
|
for (int colIndex = 0; colIndex < columns.size(); colIndex++) {
|
||||||
String column = columns.get(colIndex);
|
String column = columns.get(colIndex);
|
||||||
@@ -1062,6 +1195,23 @@ public class BrandTaskService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!undetectedRows.isEmpty()) {
|
||||||
|
var undetectedSheet = workbook.createSheet("未检测品牌");
|
||||||
|
var undetectedHeader = undetectedSheet.createRow(0);
|
||||||
|
for (int i = 0; i < columns.size(); i++) {
|
||||||
|
undetectedHeader.createCell(i).setCellValue(columns.get(i));
|
||||||
|
}
|
||||||
|
for (int i = 0; i < undetectedRows.size(); i++) {
|
||||||
|
Map<String, Object> rowData = undetectedRows.get(i);
|
||||||
|
var row = undetectedSheet.createRow(i + 1);
|
||||||
|
for (int colIndex = 0; colIndex < columns.size(); colIndex++) {
|
||||||
|
String column = columns.get(colIndex);
|
||||||
|
row.createCell(colIndex).setCellValue(Objects.toString(rowData.getOrDefault(column, ""), ""));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
applyBrandSheetWidths(undetectedSheet, columns.size());
|
||||||
|
}
|
||||||
|
|
||||||
var invalidSheet = workbook.createSheet("不符合品牌");
|
var invalidSheet = workbook.createSheet("不符合品牌");
|
||||||
var invalidHeader = invalidSheet.createRow(0);
|
var invalidHeader = invalidSheet.createRow(0);
|
||||||
invalidHeader.createCell(0).setCellValue("品牌");
|
invalidHeader.createCell(0).setCellValue("品牌");
|
||||||
@@ -1387,9 +1537,58 @@ public class BrandTaskService {
|
|||||||
failStaleRunningTask(task.getId(), "前端长时间无响应,任务已自动失败");
|
failStaleRunningTask(task.getId(), "前端长时间无响应,任务已自动失败");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
failNoUploadStaleRunningTasks();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 二次判死:前端心跳正常但连续 N 分钟无结果上报(治「心跳续命」的假活任务)。
|
||||||
|
*
|
||||||
|
* <p>既有心跳线候选条件是 updated_at/last_heartbeat_at 陈旧,而前端心跳会持续刷新它们——
|
||||||
|
* 主线程卡死时心跳线程照发,任务永远命不中(与生产 28131 同型缺口)。
|
||||||
|
* 本线候选取「心跳新鲜 + 创建超过 N 分钟」,判据用结果上报时写入 progress hash 的
|
||||||
|
* last_result_at(仅 saveProgressFromResult 写);从未上报(无该字段)跳过,避免误杀首批较慢的正常任务。
|
||||||
|
*/
|
||||||
|
private void failNoUploadStaleRunningTasks() {
|
||||||
|
long minutes = brandProgressProperties.getNoResultUploadTimeoutMinutes();
|
||||||
|
if (minutes <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
LocalDateTime heartbeatThreshold = LocalDateTime.now()
|
||||||
|
.minusMinutes(brandProgressProperties.getHeartbeatTimeoutMinutes());
|
||||||
|
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(minutes);
|
||||||
|
List<BrandCrawlTaskEntity> runningTasks = brandCrawlTaskMapper.selectList(new LambdaQueryWrapper<BrandCrawlTaskEntity>()
|
||||||
|
.eq(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING)
|
||||||
|
.ge(BrandCrawlTaskEntity::getUpdatedAt, heartbeatThreshold)
|
||||||
|
.lt(BrandCrawlTaskEntity::getCreatedAt, cutoff));
|
||||||
|
for (BrandCrawlTaskEntity task : runningTasks) {
|
||||||
|
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(task.getId(), 0L);
|
||||||
|
if (taskLockHandle == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try (taskLockHandle) {
|
||||||
|
Map<Object, Object> progress = brandTaskProgressCacheService.getProgress(task.getId());
|
||||||
|
long lastResultAt = 0L;
|
||||||
|
try {
|
||||||
|
lastResultAt = Long.parseLong(String.valueOf(progress.getOrDefault("last_result_at", "0")));
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
}
|
||||||
|
if (lastResultAt <= 0L) {
|
||||||
|
// 从未上报结果:保守跳过(首批可能较慢)
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
LocalDateTime lastResult = LocalDateTime.ofInstant(Instant.ofEpochMilli(lastResultAt), ZoneId.systemDefault());
|
||||||
|
if (lastResult.isAfter(cutoff)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
log.warn("[brand-stale-check] no-upload failing taskId={} lastResultAt={} timeoutMinutes={}",
|
||||||
|
task.getId(), lastResult, minutes);
|
||||||
|
failStaleRunningTask(task.getId(),
|
||||||
|
"连续 " + minutes + " 分钟无结果回传,疑似客户端任务卡死,任务已自动失败(最后结果上报 " + lastResult + ")");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private TaskDistributedLockService.LockHandle acquireTaskLock(Long taskId, long waitMillis) {
|
private TaskDistributedLockService.LockHandle acquireTaskLock(Long taskId, long waitMillis) {
|
||||||
TaskDistributedLockService.LockHandle lockHandle = taskDistributedLockService.acquire(MODULE_TYPE, taskId, waitMillis);
|
TaskDistributedLockService.LockHandle lockHandle = taskDistributedLockService.acquire(MODULE_TYPE, taskId, waitMillis);
|
||||||
if (lockHandle == null) {
|
if (lockHandle == null) {
|
||||||
|
|||||||
+71
-11
@@ -13,9 +13,12 @@ import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
|||||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.dao.DuplicateKeyException;
|
import org.springframework.dao.DuplicateKeyException;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
import org.springframework.transaction.support.TransactionSynchronization;
|
||||||
|
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||||
|
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.security.MessageDigest;
|
import java.security.MessageDigest;
|
||||||
@@ -27,6 +30,7 @@ import java.util.Map;
|
|||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
public class BrandTaskStorageService {
|
public class BrandTaskStorageService {
|
||||||
|
|
||||||
private static final String MODULE_TYPE = "BRAND";
|
private static final String MODULE_TYPE = "BRAND";
|
||||||
@@ -236,6 +240,13 @@ public class BrandTaskStorageService {
|
|||||||
return new ChunkStoreResult(false, completed, countCompletedFiles(taskId), aggregate);
|
return new ChunkStoreResult(false, completed, countCompletedFiles(taskId), aggregate);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 删除任务的全部范围/分片数据。
|
||||||
|
*
|
||||||
|
* <p>事务边界:远端载荷删除移到事务提交后执行 —— 单次删除超时 120s、重试 3 次,
|
||||||
|
* 放在 DB 事务里会让连接被长时间占用(项目在 TaskScopePayloadStorageService 已写明该约束)。
|
||||||
|
* 提交后再删还有个好处:引用计数查询看到的是删除完成后的 DB 状态,判断更准确。
|
||||||
|
*/
|
||||||
@Transactional
|
@Transactional
|
||||||
public void deleteTaskData(Long taskId) {
|
public void deleteTaskData(Long taskId) {
|
||||||
if (taskId == null || taskId <= 0) {
|
if (taskId == null || taskId <= 0) {
|
||||||
@@ -245,27 +256,68 @@ public class BrandTaskStorageService {
|
|||||||
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
|
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
|
||||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
||||||
if (states != null) {
|
|
||||||
for (TaskScopeStateEntity state : states) {
|
|
||||||
transientPayloadStorageService.deletePayloadIfPresent(state.getParsedPayloadJson());
|
|
||||||
transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
.select(TaskChunkEntity::getPayloadJson)
|
.select(TaskChunkEntity::getPayloadJson)
|
||||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||||
if (chunks != null) {
|
|
||||||
for (TaskChunkEntity chunk : chunks) {
|
|
||||||
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
|
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||||
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
||||||
|
deletePayloadsAfterCommit(states, chunks, taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 事务提交后逐条删远端载荷;无事务时立即执行(与 TaskResultItemService 同款兜底)。 */
|
||||||
|
private void deletePayloadsAfterCommit(List<TaskScopeStateEntity> states,
|
||||||
|
List<TaskChunkEntity> chunks,
|
||||||
|
Long taskId) {
|
||||||
|
List<String> payloads = new ArrayList<>();
|
||||||
|
if (states != null) {
|
||||||
|
for (TaskScopeStateEntity state : states) {
|
||||||
|
if (state == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (state.getParsedPayloadJson() != null && !state.getParsedPayloadJson().isBlank()) {
|
||||||
|
payloads.add(state.getParsedPayloadJson());
|
||||||
|
}
|
||||||
|
if (state.getStateJson() != null && !state.getStateJson().isBlank()) {
|
||||||
|
payloads.add(state.getStateJson());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (chunks != null) {
|
||||||
|
for (TaskChunkEntity chunk : chunks) {
|
||||||
|
if (chunk != null && chunk.getPayloadJson() != null && !chunk.getPayloadJson().isBlank()) {
|
||||||
|
payloads.add(chunk.getPayloadJson());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (payloads.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (TransactionSynchronizationManager.isSynchronizationActive()) {
|
||||||
|
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
|
||||||
|
@Override
|
||||||
|
public void afterCommit() {
|
||||||
|
deletePayloadsNow(payloads, taskId);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
deletePayloadsNow(payloads, taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void deletePayloadsNow(List<String> payloads, Long taskId) {
|
||||||
|
for (String payload : payloads) {
|
||||||
|
try {
|
||||||
|
transientPayloadStorageService.deletePayloadIfPresent(payload);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
// 事务已提交,远端删除失败只记日志,不影响任务数据清理结果
|
||||||
|
log.warn("[brand-storage] 载荷删除失败 taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void saveAggregate(Long taskId,
|
private void saveAggregate(Long taskId,
|
||||||
@@ -325,6 +377,7 @@ public class BrandTaskStorageService {
|
|||||||
aggregate.setCompleted(false);
|
aggregate.setCompleted(false);
|
||||||
aggregate.setInvalidBrands(new ArrayList<>());
|
aggregate.setInvalidBrands(new ArrayList<>());
|
||||||
aggregate.setQueryFailedBrands(new ArrayList<>());
|
aggregate.setQueryFailedBrands(new ArrayList<>());
|
||||||
|
aggregate.setKeptBrands(new ArrayList<>());
|
||||||
return aggregate;
|
return aggregate;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -362,6 +415,12 @@ public class BrandTaskStorageService {
|
|||||||
if (file.getQueryFailedBrands() != null && !file.getQueryFailedBrands().isEmpty()) {
|
if (file.getQueryFailedBrands() != null && !file.getQueryFailedBrands().isEmpty()) {
|
||||||
aggregate.getQueryFailedBrands().addAll(file.getQueryFailedBrands());
|
aggregate.getQueryFailedBrands().addAll(file.getQueryFailedBrands());
|
||||||
}
|
}
|
||||||
|
if (file.getKeptRows() != null && !file.getKeptRows().isEmpty()) {
|
||||||
|
if (aggregate.getKeptBrands() == null) {
|
||||||
|
aggregate.setKeptBrands(new ArrayList<>());
|
||||||
|
}
|
||||||
|
aggregate.getKeptBrands().addAll(file.getKeptRows());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private BrandFileAggregateCacheDto rebuildAggregate(Long taskId, String scopeKey, String scopeHash) {
|
private BrandFileAggregateCacheDto rebuildAggregate(Long taskId, String scopeKey, String scopeHash) {
|
||||||
@@ -379,6 +438,7 @@ public class BrandTaskStorageService {
|
|||||||
aggregate.setCompleted(false);
|
aggregate.setCompleted(false);
|
||||||
aggregate.setInvalidBrands(new ArrayList<>());
|
aggregate.setInvalidBrands(new ArrayList<>());
|
||||||
aggregate.setQueryFailedBrands(new ArrayList<>());
|
aggregate.setQueryFailedBrands(new ArrayList<>());
|
||||||
|
aggregate.setKeptBrands(new ArrayList<>());
|
||||||
}
|
}
|
||||||
return aggregate;
|
return aggregate;
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-2
@@ -121,8 +121,8 @@ public class CollectDataController {
|
|||||||
|
|
||||||
@PostMapping("/tasks/progress/light")
|
@PostMapping("/tasks/progress/light")
|
||||||
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
|
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
|
||||||
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt/rowsVersion),"
|
||||||
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
+ "不返回行明细内容(仅多一次结果行版本聚合查询),响应体更小;旧 progress/batch 端点保留不动。")
|
||||||
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||||
// 透传 userId:此前传 null = 不过滤归属,匿名遍历 taskId 即可读他人任务状态(2026-09 审查)
|
// 透传 userId:此前传 null = 不过滤归属,匿名遍历 taskId 即可读他人任务状态(2026-09 审查)
|
||||||
return ApiResponse.success(service.progressLight(request.getTaskIds(), request.getUserId()));
|
return ApiResponse.success(service.progressLight(request.getTaskIds(), request.getUserId()));
|
||||||
|
|||||||
+241
-93
@@ -51,6 +51,7 @@ import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
|||||||
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||||
@@ -179,6 +180,10 @@ public class CollectDataService {
|
|||||||
@Value("${aiimage.collect-data.stale-timeout-minutes:30}")
|
@Value("${aiimage.collect-data.stale-timeout-minutes:30}")
|
||||||
private long staleTimeoutMinutes;
|
private long staleTimeoutMinutes;
|
||||||
|
|
||||||
|
/** 二次判死线(心跳正常但连续 N 分钟无结果分片上报)阈值,分钟;<=0 关闭本线。 */
|
||||||
|
@Value("${aiimage.collect-data.no-result-upload-timeout-minutes:180}")
|
||||||
|
private long noResultUploadTimeoutMinutes;
|
||||||
|
|
||||||
@Value("${aiimage.collect-data.max-source-file-bytes:0}")
|
@Value("${aiimage.collect-data.max-source-file-bytes:0}")
|
||||||
private Long maxSourceFileBytes;
|
private Long maxSourceFileBytes;
|
||||||
|
|
||||||
@@ -381,14 +386,18 @@ public class CollectDataService {
|
|||||||
@Transactional
|
@Transactional
|
||||||
public void activateTask(Long taskId, Long userId) {
|
public void activateTask(Long taskId, Long userId) {
|
||||||
FileTaskEntity task = requireTask(taskId, userId);
|
FileTaskEntity task = requireTask(taskId, userId);
|
||||||
// 条件更新:上面的「已结束」判断与写入之间存在窗口(TOCTOU),期间 /fail 可能已把任务
|
// 只允许 PENDING→RUNNING(条件更新):既堵住 TOCTOU(/fail 抢先标 FAILED 后被整行
|
||||||
// 标为 FAILED —— 整行 updateById 会把它复活成 RUNNING(前端显示"执行中"但无人推进)
|
// updateById 复活成 RUNNING),又与客户端「兜底拉取」的原子认领互斥,谁先翻转谁执行
|
||||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||||
.eq(FileTaskEntity::getId, task.getId())
|
.eq(FileTaskEntity::getId, task.getId())
|
||||||
.notIn(FileTaskEntity::getStatus, STATUS_SUCCESS, STATUS_FAILED)
|
.eq(FileTaskEntity::getStatus, STATUS_PENDING)
|
||||||
.set(FileTaskEntity::getStatus, STATUS_RUNNING)
|
.set(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
||||||
if (updated == 0) {
|
if (updated == 0) {
|
||||||
|
FileTaskEntity latest = fileTaskMapper.selectById(task.getId());
|
||||||
|
if (latest != null && STATUS_RUNNING.equals(latest.getStatus())) {
|
||||||
|
throw new BusinessException("任务已在执行中(可能已由客户端自动接管),无需重复启动");
|
||||||
|
}
|
||||||
throw new BusinessException("任务已结束");
|
throw new BusinessException("任务已结束");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -422,70 +431,85 @@ public class CollectDataService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Transactional
|
/**
|
||||||
|
* 进度心跳。
|
||||||
|
*
|
||||||
|
* <p>事务边界:Redis 任务锁在事务外获取(自旋等待最长 TASK_LOCK_WAIT_MILLIS,
|
||||||
|
* 放在 @Transactional 里会白占一个 Hikari 连接),DB 段(统计持久化 + 任务行更新)
|
||||||
|
* 仍在一个事务内。
|
||||||
|
*/
|
||||||
public void updateProgress(Long taskId, TaskHeartbeatRequest request) {
|
public void updateProgress(Long taskId, TaskHeartbeatRequest request) {
|
||||||
if (taskId == null || taskId <= 0 || request == null) {
|
if (taskId == null || taskId <= 0 || request == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
|
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
|
||||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
if (transactionTemplate == null) {
|
||||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) {
|
// 单测场景(@InjectMocks 未注入事务模板):退化为直接执行 DB 段
|
||||||
|
updateProgressLocked(taskId, request);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
CollectDataStats stats = loadStats(task);
|
transactionTemplate.executeWithoutResult(status -> updateProgressLocked(taskId, request));
|
||||||
boolean changed = false;
|
|
||||||
Integer current = request.getCurrent();
|
|
||||||
Integer total = request.getTotal();
|
|
||||||
if (current != null && total != null && total > 0) {
|
|
||||||
int totalRows = stats.totalRows > 0 ? stats.totalRows : total;
|
|
||||||
int processedRows = Math.max(stats.processedRows, Math.min(Math.max(current, 0), totalRows));
|
|
||||||
if (stats.totalRows != totalRows || stats.processedRows != processedRows) {
|
|
||||||
stats.totalRows = totalRows;
|
|
||||||
stats.processedRows = processedRows;
|
|
||||||
changed = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (request.getCollectStage() != null && !java.util.Objects.equals(stats.collectStage, request.getCollectStage())) {
|
|
||||||
stats.collectStage = request.getCollectStage();
|
|
||||||
changed = true;
|
|
||||||
}
|
|
||||||
if (request.getCurrentKeyword() != null && !java.util.Objects.equals(stats.currentKeyword, request.getCurrentKeyword())) {
|
|
||||||
stats.currentKeyword = request.getCurrentKeyword();
|
|
||||||
changed = true;
|
|
||||||
}
|
|
||||||
if (request.getSearchCurrentPage() != null && stats.searchCurrentPage != Math.max(0, request.getSearchCurrentPage())) {
|
|
||||||
stats.searchCurrentPage = Math.max(0, request.getSearchCurrentPage());
|
|
||||||
changed = true;
|
|
||||||
}
|
|
||||||
if (request.getSearchTotalPages() != null && stats.searchTotalPages != Math.max(0, request.getSearchTotalPages())) {
|
|
||||||
stats.searchTotalPages = Math.max(0, request.getSearchTotalPages());
|
|
||||||
changed = true;
|
|
||||||
}
|
|
||||||
if (request.getDetailProcessedAsins() != null && stats.detailProcessedAsins != Math.max(0, request.getDetailProcessedAsins())) {
|
|
||||||
stats.detailProcessedAsins = Math.max(0, request.getDetailProcessedAsins());
|
|
||||||
changed = true;
|
|
||||||
}
|
|
||||||
if (request.getDetailTotalAsins() != null && stats.detailTotalAsins != Math.max(0, request.getDetailTotalAsins())) {
|
|
||||||
stats.detailTotalAsins = Math.max(0, request.getDetailTotalAsins());
|
|
||||||
changed = true;
|
|
||||||
}
|
|
||||||
if (!changed) {
|
|
||||||
// 内容未变化:仅当脏数据兜底窗口到期时才强制刷新一次,
|
|
||||||
// 否则零 UPDATE(重复心跳幂等)。
|
|
||||||
if (!shouldForceProgressFlush()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
} else if (shouldThrottleProgressFlush()) {
|
|
||||||
// 节流窗口内:合并写(只更新内存态、不落库),窗口到期后统一持久化。
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
persistStats(task, stats);
|
|
||||||
task.setUpdatedAt(LocalDateTime.now());
|
|
||||||
fileTaskMapper.updateById(task);
|
|
||||||
lastProgressFlushMillis = System.currentTimeMillis();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void updateProgressLocked(Long taskId, TaskHeartbeatRequest request) {
|
||||||
|
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||||
|
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
CollectDataStats stats = loadStats(task);
|
||||||
|
boolean changed = false;
|
||||||
|
Integer current = request.getCurrent();
|
||||||
|
Integer total = request.getTotal();
|
||||||
|
if (current != null && total != null && total > 0) {
|
||||||
|
int totalRows = stats.totalRows > 0 ? stats.totalRows : total;
|
||||||
|
int processedRows = Math.max(stats.processedRows, Math.min(Math.max(current, 0), totalRows));
|
||||||
|
if (stats.totalRows != totalRows || stats.processedRows != processedRows) {
|
||||||
|
stats.totalRows = totalRows;
|
||||||
|
stats.processedRows = processedRows;
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (request.getCollectStage() != null && !java.util.Objects.equals(stats.collectStage, request.getCollectStage())) {
|
||||||
|
stats.collectStage = request.getCollectStage();
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
if (request.getCurrentKeyword() != null && !java.util.Objects.equals(stats.currentKeyword, request.getCurrentKeyword())) {
|
||||||
|
stats.currentKeyword = request.getCurrentKeyword();
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
if (request.getSearchCurrentPage() != null && stats.searchCurrentPage != Math.max(0, request.getSearchCurrentPage())) {
|
||||||
|
stats.searchCurrentPage = Math.max(0, request.getSearchCurrentPage());
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
if (request.getSearchTotalPages() != null && stats.searchTotalPages != Math.max(0, request.getSearchTotalPages())) {
|
||||||
|
stats.searchTotalPages = Math.max(0, request.getSearchTotalPages());
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
if (request.getDetailProcessedAsins() != null && stats.detailProcessedAsins != Math.max(0, request.getDetailProcessedAsins())) {
|
||||||
|
stats.detailProcessedAsins = Math.max(0, request.getDetailProcessedAsins());
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
if (request.getDetailTotalAsins() != null && stats.detailTotalAsins != Math.max(0, request.getDetailTotalAsins())) {
|
||||||
|
stats.detailTotalAsins = Math.max(0, request.getDetailTotalAsins());
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
if (!changed) {
|
||||||
|
// 内容未变化:仅当脏数据兜底窗口到期时才强制刷新一次,
|
||||||
|
// 否则零 UPDATE(重复心跳幂等)。
|
||||||
|
if (!shouldForceProgressFlush()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} else if (shouldThrottleProgressFlush()) {
|
||||||
|
// 节流窗口内:合并写(只更新内存态、不落库),窗口到期后统一持久化。
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
persistStats(task, stats);
|
||||||
|
task.setUpdatedAt(LocalDateTime.now());
|
||||||
|
fileTaskMapper.updateById(task);
|
||||||
|
lastProgressFlushMillis = System.currentTimeMillis();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 节流判定:progressThrottleMillis>0 且距上次实际落库未超过窗口 → 合并写(跳过 UPDATE)。
|
* 节流判定:progressThrottleMillis>0 且距上次实际落库未超过窗口 → 合并写(跳过 UPDATE)。
|
||||||
* progressThrottleMillis<=0 视为关闭节流,恒返回 false(每次心跳都落库,兼容旧行为)。
|
* progressThrottleMillis<=0 视为关闭节流,恒返回 false(每次心跳都落库,兼容旧行为)。
|
||||||
@@ -522,6 +546,7 @@ public class CollectDataService {
|
|||||||
for (FileTaskEntity task : tasks) {
|
for (FileTaskEntity task : tasks) {
|
||||||
finalizeStaleTask(task.getId(), threshold, timeoutMinutes);
|
finalizeStaleTask(task.getId(), threshold, timeoutMinutes);
|
||||||
}
|
}
|
||||||
|
finalizeNoUploadStaleTasks(threshold);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -561,6 +586,86 @@ public class CollectDataService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 二次判死:心跳正常但连续 N 分钟无结果分片上报(治「心跳续命」的假活任务)。
|
||||||
|
*
|
||||||
|
* <p>既有心跳线候选条件是 updated_at 陈旧,而客户端心跳会持续刷新 updated_at——
|
||||||
|
* 主线程卡死(浏览器自动化等待/异常)时心跳线程照发,任务永远命不中。
|
||||||
|
* 本线候选取「心跳新鲜(既有线放过)+ 创建超过 N 分钟」,判据用
|
||||||
|
* biz_task_scope_state.last_chunk_at(只随结果分片上报刷新);
|
||||||
|
* 从未上报(无 scope 行或全 NULL)的任务跳过,避免误杀首批较慢的正常任务。
|
||||||
|
*/
|
||||||
|
private void finalizeNoUploadStaleTasks(LocalDateTime heartbeatThreshold) {
|
||||||
|
long minutes = noResultUploadTimeoutMinutes;
|
||||||
|
if (minutes <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(minutes);
|
||||||
|
List<FileTaskEntity> candidates = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
|
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||||
|
.ge(FileTaskEntity::getUpdatedAt, heartbeatThreshold)
|
||||||
|
.lt(FileTaskEntity::getCreatedAt, cutoff)
|
||||||
|
.orderByAsc(FileTaskEntity::getCreatedAt)
|
||||||
|
.last("limit 200"));
|
||||||
|
if (candidates.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Map<Long, LocalDateTime> lastResultAtByTaskId = new HashMap<>();
|
||||||
|
for (TaskScopeLastChunkDto dto : taskScopeStateMapper.selectLastChunkAtByTaskIds(
|
||||||
|
candidates.stream().map(FileTaskEntity::getId).toList())) {
|
||||||
|
if (dto.taskId() != null && dto.lastChunkAt() != null) {
|
||||||
|
lastResultAtByTaskId.put(dto.taskId(), dto.lastChunkAt());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (FileTaskEntity task : candidates) {
|
||||||
|
LocalDateTime lastResultAt = lastResultAtByTaskId.get(task.getId());
|
||||||
|
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
finalizeNoUploadStaleTask(task.getId(), lastResultAt, minutes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 单个任务的二次判死收尾:锁内复查心跳后,有分片→组装部分工作簿;无分片→标失败。 */
|
||||||
|
private void finalizeNoUploadStaleTask(Long taskId, LocalDateTime lastResultAt, long minutes) {
|
||||||
|
try (TaskDistributedLockService.LockHandle lock =
|
||||||
|
taskDistributedLockService.acquire(MODULE_TYPE, taskId, 0L)) {
|
||||||
|
if (lock == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||||
|
LocalDateTime heartbeatFreshAfter = LocalDateTime.now().minusMinutes(Math.max(1L, staleTimeoutMinutes));
|
||||||
|
if (task == null
|
||||||
|
|| !MODULE_TYPE.equals(task.getModuleType())
|
||||||
|
|| !STATUS_RUNNING.equals(task.getStatus())
|
||||||
|
|| task.getUpdatedAt() == null
|
||||||
|
|| task.getUpdatedAt().isBefore(heartbeatFreshAfter)) {
|
||||||
|
// 已终态、或心跳在排队期间回落到陈旧(交回既有心跳线处理)
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (taskFileJobService.countUnfinishedAssembleJobs(taskId, MODULE_TYPE) > 0L) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
FileResultEntity result = ensureTaskResult(task);
|
||||||
|
CollectDataStats stats = loadStats(task);
|
||||||
|
if (hasReceivedChunks(taskId)) {
|
||||||
|
enqueueFinalWorkbook(task, result, stats);
|
||||||
|
log.warn("[collect-data] no-upload stale task enqueued partial workbook taskId={} lastResultAt={} timeoutMinutes={} finalRows={}",
|
||||||
|
taskId, lastResultAt, minutes, stats.finalRowCount);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
markTaskFailed(task, result,
|
||||||
|
"连续 " + minutes + " 分钟无结果回传,疑似客户端任务卡死,任务已自动失败(最后结果上报 " + lastResultAt + ")",
|
||||||
|
stats);
|
||||||
|
log.warn("[collect-data] no-upload stale task failed without chunks taskId={} lastResultAt={} timeoutMinutes={}",
|
||||||
|
taskId, lastResultAt, minutes);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[collect-data] no-upload stale task finalization failed taskId={} msg={}",
|
||||||
|
taskId, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public CollectDataDashboardVo dashboard(Long userId) {
|
public CollectDataDashboardVo dashboard(Long userId) {
|
||||||
CollectDataDashboardVo vo = new CollectDataDashboardVo();
|
CollectDataDashboardVo vo = new CollectDataDashboardVo();
|
||||||
vo.setPendingTaskCount(countActiveTasks(userId));
|
vo.setPendingTaskCount(countActiveTasks(userId));
|
||||||
@@ -677,6 +782,39 @@ public class CollectDataService {
|
|||||||
throw new BusinessException("request is empty");
|
throw new BusinessException("request is empty");
|
||||||
}
|
}
|
||||||
ensureRustfsPayloadStorageEnabled();
|
ensureRustfsPayloadStorageEnabled();
|
||||||
|
|
||||||
|
// 锁外预检:任务不存在/已结束时立即失败,不为终态任务白跑去重查询与品牌检测。
|
||||||
|
// 只做快速失败,并发正确性仍由锁内的重读复核保证。
|
||||||
|
FileTaskEntity probe = fileTaskMapper.selectById(taskId);
|
||||||
|
if (probe == null || !MODULE_TYPE.equals(probe.getModuleType())) {
|
||||||
|
throw new BusinessException("任务不存在");
|
||||||
|
}
|
||||||
|
if (STATUS_SUCCESS.equals(probe.getStatus()) || STATUS_FAILED.equals(probe.getStatus())) {
|
||||||
|
log.warn("[collect-data] 任务已结束,拒绝重复提交 taskId={} status={}", taskId, probe.getStatus());
|
||||||
|
throw new BusinessException(40901, "任务已结束,拒绝重复提交");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 归一化 / 去重过滤 / 品牌检测放在锁外:品牌检测是同步远程调用,上游 16890 抖动时
|
||||||
|
// 单品牌 10 次重试合计上百秒(taskId 28599 实测:chunk 回传在锁内等品牌检测 103.5 秒,
|
||||||
|
// 期间心跳与客户端重试全部撞 40902 拿不到锁,客户端 5 次重试预算耗尽后中止整个采集)。
|
||||||
|
// 这几步只依赖本批入参、不写任务状态,放锁外不改变 chunk 落库的串行语义。
|
||||||
|
List<CollectDataResultRowVo> rows = normalizeSubmitRows(request.getItems());
|
||||||
|
buildParseLimits().validateChunkRowCount(rows.size());
|
||||||
|
// 「结果文件」sheet 需要 Python 回传的全量数据(不经任何后端过滤),
|
||||||
|
// 因此 chunk payload 写入未筛除空 ASIN 的全量 rows;
|
||||||
|
// 仅当 ASIN 非空时才进入去重 / 无效品牌 / 品牌检测链路并落 biz_task_result_item。
|
||||||
|
List<CollectDataResultRowVo> rowsForFiltering = new ArrayList<>(rows.size());
|
||||||
|
for (CollectDataResultRowVo row : rows) {
|
||||||
|
if (row.getAsin() != null && !row.getAsin().isBlank()) {
|
||||||
|
rowsForFiltering.add(row);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
long prepareStartAt = System.currentTimeMillis();
|
||||||
|
CollectDataBatchQuery.FilterResult filtered = collectDataBatchQuery.filter(rowsForFiltering);
|
||||||
|
CollectDataBrandBatchFilter.BrandBatchOutcome brandOutcome = brandBatchFilter.filter(filtered.kept());
|
||||||
|
log.info("[collect-data] 锁外预处理完成 taskId={} rows={} 去重后={} 品牌检测耗时={}ms",
|
||||||
|
taskId, rows.size(), filtered.kept().size(), System.currentTimeMillis() - prepareStartAt);
|
||||||
|
|
||||||
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
|
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
|
||||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||||
@@ -714,25 +852,23 @@ public class CollectDataService {
|
|||||||
return buildSubmitVo(task, result, chunkIndex, chunkTotal, request, 0);
|
return buildSubmitVo(task, result, chunkIndex, chunkTotal, request, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
List<CollectDataResultRowVo> rows = normalizeSubmitRows(request.getItems());
|
|
||||||
buildParseLimits().validateChunkRowCount(rows.size());
|
|
||||||
CollectDataStats stats = loadStats(task);
|
CollectDataStats stats = loadStats(task);
|
||||||
stats.receivedRows += rows.size();
|
stats.receivedRows += rows.size();
|
||||||
stats.currentChunkRows = rows.size();
|
stats.currentChunkRows = rows.size();
|
||||||
|
|
||||||
// 「结果文件」sheet 需要 Python 回传的全量数据(不经任何后端过滤),
|
|
||||||
// 因此 chunk payload 写入未筛除空 ASIN 的全量 rows;
|
|
||||||
// 仅当 ASIN 非空时才进入去重 / 无效品牌 / 品牌检测链路并落 biz_task_result_item。
|
|
||||||
List<CollectDataResultRowVo> rowsForFiltering = new ArrayList<>(rows.size());
|
|
||||||
for (CollectDataResultRowVo row : rows) {
|
|
||||||
if (row.getAsin() != null && !row.getAsin().isBlank()) {
|
|
||||||
rowsForFiltering.add(row);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
CollectDataBatchQuery.FilterResult filtered = collectDataBatchQuery.filter(rowsForFiltering);
|
|
||||||
stats.dedupeFilteredCount += filtered.dedupeFilteredCount();
|
stats.dedupeFilteredCount += filtered.dedupeFilteredCount();
|
||||||
stats.invalidFilteredCount += filtered.invalidFilteredCount();
|
stats.invalidFilteredCount += filtered.invalidFilteredCount();
|
||||||
List<CollectDataResultRowVo> accepted = filterByBrandCheck(filtered.kept(), stats);
|
stats.brandRejectedCount += brandOutcome.rejected().size();
|
||||||
|
stats.brandQueryFailedCount += brandOutcome.queryFailed().size();
|
||||||
|
// 只有被服务端判定为无效品牌(rejected)的行才写入无效品牌表;
|
||||||
|
// queryFailed 是品牌检测服务端到端失败(如缺 X-Token 422/超时/限流),
|
||||||
|
// 一并写入会把故障期间被误伤的品牌永久拉黑,后续同品牌商品全部被
|
||||||
|
// invalidFiltered 过滤(task-27265 实测:通过 Python 过滤的 21 行中
|
||||||
|
// 8 行查询失败被写表后误杀,最终结果 Excel 0 行)。rejected 为空时
|
||||||
|
// 不触发任何写入,避免空批次无意义调用。
|
||||||
|
if (!brandOutcome.rejected().isEmpty()) {
|
||||||
|
invalidAsinBatchWriter.writeBatch(brandOutcome.rejected());
|
||||||
|
}
|
||||||
|
List<CollectDataResultRowVo> accepted = brandOutcome.accepted();
|
||||||
// 结果明细改为 chunk 级存储:整个 chunk 的 accepted 行共享一个
|
// 结果明细改为 chunk 级存储:整个 chunk 的 accepted 行共享一个
|
||||||
// RustFS 对象(deterministic key,同 chunk 重提覆盖同一对象),
|
// RustFS 对象(deterministic key,同 chunk 重提覆盖同一对象),
|
||||||
// biz_task_result_item.payload_json 只存 {chunk, offset, payload} 引用。
|
// biz_task_result_item.payload_json 只存 {chunk, offset, payload} 引用。
|
||||||
@@ -767,6 +903,14 @@ public class CollectDataService {
|
|||||||
|
|
||||||
if (request.getError() != null && !request.getError().isBlank()) {
|
if (request.getError() != null && !request.getError().isBlank()) {
|
||||||
markTaskFailed(task, result, request.getError(), stats);
|
markTaskFailed(task, result, request.getError(), stats);
|
||||||
|
// 失败但已收到分片:照常组装结果文件,让用户能下载已采集的数据。
|
||||||
|
// 此前失败分支只标失败不组装,已落库的数据也没有任何结果文件可下载
|
||||||
|
// (taskId 28599:55 个分片全部收到、187 行明细已落库,用户却拿不到文件)。
|
||||||
|
if (hasReceivedChunks(taskId)) {
|
||||||
|
enqueueFinalWorkbook(task, result, stats);
|
||||||
|
log.warn("[collect-data] 任务失败仍组装部分结果 taskId={} error={} finalRows={}",
|
||||||
|
taskId, request.getError(), stats.finalRowCount);
|
||||||
|
}
|
||||||
} else if (Boolean.TRUE.equals(request.getDone())) {
|
} else if (Boolean.TRUE.equals(request.getDone())) {
|
||||||
enqueueFinalWorkbook(task, result, stats);
|
enqueueFinalWorkbook(task, result, stats);
|
||||||
} else {
|
} else {
|
||||||
@@ -828,22 +972,6 @@ public class CollectDataService {
|
|||||||
return rows;
|
return rows;
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<CollectDataResultRowVo> filterByBrandCheck(List<CollectDataResultRowVo> rows, CollectDataStats stats) {
|
|
||||||
CollectDataBrandBatchFilter.BrandBatchOutcome outcome = brandBatchFilter.filter(rows);
|
|
||||||
stats.brandRejectedCount += outcome.rejected().size();
|
|
||||||
stats.brandQueryFailedCount += outcome.queryFailed().size();
|
|
||||||
// 只有被服务端判定为无效品牌(rejected)的行才写入无效品牌表;
|
|
||||||
// queryFailed 是品牌检测服务端到端失败(如缺 X-Token 422/超时/限流),
|
|
||||||
// 一并写入会把故障期间被误伤的品牌永久拉黑,后续同品牌商品全部被
|
|
||||||
// invalidFiltered 过滤(task-27265 实测:通过 Python 过滤的 21 行中
|
|
||||||
// 8 行查询失败被写表后误杀,最终结果 Excel 0 行)。rejected 为空时
|
|
||||||
// 不触发任何写入,避免空批次无意义调用。
|
|
||||||
if (!outcome.rejected().isEmpty()) {
|
|
||||||
invalidAsinBatchWriter.writeBatch(outcome.rejected());
|
|
||||||
}
|
|
||||||
return outcome.accepted();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void persistChunk(Long taskId,
|
private void persistChunk(Long taskId,
|
||||||
String scopeKey,
|
String scopeKey,
|
||||||
String scopeHash,
|
String scopeHash,
|
||||||
@@ -906,7 +1034,8 @@ public class CollectDataService {
|
|||||||
result.setResultFileSize(0L);
|
result.setResultFileSize(0L);
|
||||||
result.setResultContentType(CONTENT_TYPE_XLSX);
|
result.setResultContentType(CONTENT_TYPE_XLSX);
|
||||||
result.setRowCount(stats.finalRowCount);
|
result.setRowCount(stats.finalRowCount);
|
||||||
result.setErrorMessage(null);
|
// 不清 errorMessage:失败任务的部分结果组装也走这里,清掉会让用户看不到真实失败原因
|
||||||
|
// (成功路径的 errorMessage 本来就为 null,无需清理)。
|
||||||
fileResultMapper.updateById(result);
|
fileResultMapper.updateById(result);
|
||||||
task.setUpdatedAt(LocalDateTime.now());
|
task.setUpdatedAt(LocalDateTime.now());
|
||||||
fileTaskMapper.updateById(task);
|
fileTaskMapper.updateById(task);
|
||||||
@@ -955,6 +1084,22 @@ public class CollectDataService {
|
|||||||
stats.summaries,
|
stats.summaries,
|
||||||
batch -> streamRawRows(task.getId(), batch));
|
batch -> streamRawRows(task.getId(), batch));
|
||||||
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
||||||
|
|
||||||
|
// 复用同一份 stats 更新 finalRowCount 后再持久化,避免重复 loadStats 丢失 summaries。
|
||||||
|
stats.finalRowCount = (int) finalRowCount;
|
||||||
|
persistStats(task, stats);
|
||||||
|
|
||||||
|
// 失败原因先留存:下面的乐观写入会清空 result.errorMessage,任务已被判失败时要用它恢复。
|
||||||
|
String failureReason = result.getErrorMessage();
|
||||||
|
if (failureReason == null || failureReason.isBlank()) {
|
||||||
|
failureReason = task.getErrorMessage();
|
||||||
|
}
|
||||||
|
if (failureReason == null || failureReason.isBlank()) {
|
||||||
|
failureReason = "任务失败,结果文件为已采集的部分数据";
|
||||||
|
}
|
||||||
|
|
||||||
|
// 结果行先按成功乐观写入:保持「结果文件先于任务成功落库」的时序,
|
||||||
|
// 万一进程在这两步之间退出,任务仍是 RUNNING,会被陈旧巡检重新组装(可自愈)。
|
||||||
result.setResultFilename(filename);
|
result.setResultFilename(filename);
|
||||||
result.setResultFileUrl(objectKey);
|
result.setResultFileUrl(objectKey);
|
||||||
result.setResultFileSize(xlsx.length());
|
result.setResultFileSize(xlsx.length());
|
||||||
@@ -964,10 +1109,7 @@ public class CollectDataService {
|
|||||||
result.setErrorMessage(null);
|
result.setErrorMessage(null);
|
||||||
fileResultMapper.updateById(result);
|
fileResultMapper.updateById(result);
|
||||||
|
|
||||||
// 复用同一份 stats 更新 finalRowCount 后再持久化,避免重复 loadStats 丢失 summaries。
|
// 条件更新:任务可能已被判失败(客户端上报失败 / 陈旧判死与结果文件组装并发)——
|
||||||
stats.finalRowCount = (int) finalRowCount;
|
|
||||||
persistStats(task, stats);
|
|
||||||
// 条件更新:任务可能已被 /fail 标为 FAILED(客户端报错与结果文件组装并发)——
|
|
||||||
// 无条件 updateById 会把 FAILED 覆盖成 SUCCESS(错误信息被清、用户看到「假成功」)。
|
// 无条件 updateById 会把 FAILED 覆盖成 SUCCESS(错误信息被清、用户看到「假成功」)。
|
||||||
// 结果文件已生成,故失败态下仍保留文件,只是不覆盖状态。
|
// 结果文件已生成,故失败态下仍保留文件,只是不覆盖状态。
|
||||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||||
@@ -980,7 +1122,13 @@ public class CollectDataService {
|
|||||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||||
if (updated == 0) {
|
if (updated == 0) {
|
||||||
log.warn("[collect-data] 结果文件已生成但任务已是 FAILED,保留失败态不覆盖 taskId={}", task.getId());
|
// 任务已是 FAILED:结果记录改回失败语义并保留真实原因,但文件 URL 照常保留,
|
||||||
|
// 用户看到「失败 + 原因」的同时仍能下载已采集的部分结果。
|
||||||
|
result.setSuccess(0);
|
||||||
|
result.setErrorMessage(failureReason);
|
||||||
|
fileResultMapper.updateById(result);
|
||||||
|
log.warn("[collect-data] 结果文件已生成但任务已是 FAILED,保留失败态与原因 taskId={} rows={} reason={}",
|
||||||
|
task.getId(), finalRowCount, failureReason);
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
FileUtil.del(xlsx);
|
FileUtil.del(xlsx);
|
||||||
|
|||||||
+71
@@ -0,0 +1,71 @@
|
|||||||
|
package com.nanri.aiimage.modules.collectdata.service;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.spi.ClientTaskPullSpi;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 集采(collect-data)的客户端兜底拉取实现。
|
||||||
|
*
|
||||||
|
* <p>Python 消费端需要 taskId / totalRows / pageSize / filters(明细行自行按 /items 分页拉取)。
|
||||||
|
* filters 取自任务行 request_json 里 parse 时落库的那份(CollectDataService#persistParsedTask),
|
||||||
|
* 序列化后就是 Python 读取的 camelCase 键(countryCode/minAmount/...)。
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class CollectDataTaskPullSpiImpl implements ClientTaskPullSpi {
|
||||||
|
|
||||||
|
private static final String QUEUE_TYPE = "collect-data-run";
|
||||||
|
private static final String TASK_TYPE = "collect-data";
|
||||||
|
|
||||||
|
private final ObjectMapper objectMapper;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String moduleType() {
|
||||||
|
return CollectDataService.MODULE_TYPE;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<String, Object> buildQueuePayload(FileTaskEntity task) {
|
||||||
|
JsonNode request = parseJson(task.getRequestJson());
|
||||||
|
if (request == null) {
|
||||||
|
log.warn("[collect-data] 兜底拉取失败:任务请求参数缺失或不可解析 taskId={}", task.getId());
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
JsonNode filtersNode = request.get("filters");
|
||||||
|
Map<String, Object> filters = filtersNode == null || filtersNode.isNull()
|
||||||
|
? Map.of()
|
||||||
|
: objectMapper.convertValue(filtersNode, Map.class);
|
||||||
|
JsonNode stats = parseJson(task.getResultJson());
|
||||||
|
int totalRows = stats == null ? 0 : stats.path("totalRows").asInt(0);
|
||||||
|
Map<String, Object> data = new LinkedHashMap<>();
|
||||||
|
data.put("taskId", task.getId());
|
||||||
|
data.put("taskNo", task.getTaskNo());
|
||||||
|
data.put("taskType", TASK_TYPE);
|
||||||
|
data.put("totalRows", totalRows);
|
||||||
|
data.put("pageSize", CollectDataService.DEFAULT_PAGE_SIZE);
|
||||||
|
data.put("filters", filters);
|
||||||
|
log.info("[collect-data] 兜底载荷已组装 taskId={} totalRows={} filters={}", task.getId(), totalRows, filters);
|
||||||
|
return Map.of("type", QUEUE_TYPE, "data", data);
|
||||||
|
}
|
||||||
|
|
||||||
|
private JsonNode parseJson(String json) {
|
||||||
|
if (json == null || json.isBlank()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return objectMapper.readTree(json);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[collect-data] 兜底拉取解析任务 JSON 失败 err={}", ex.getMessage());
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+3
-1
@@ -88,10 +88,12 @@ public class DedupeTotalDataController {
|
|||||||
@RequestParam(name = "end_date", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate,
|
@RequestParam(name = "end_date", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate,
|
||||||
@Parameter(description = "分组 ID") @RequestParam(name = "group_id", required = false) Long groupId,
|
@Parameter(description = "分组 ID") @RequestParam(name = "group_id", required = false) Long groupId,
|
||||||
@Parameter(description = "国家代码(如 DE、UK)") @RequestParam(name = "country", required = false) String country,
|
@Parameter(description = "国家代码(如 DE、UK)") @RequestParam(name = "country", required = false) String country,
|
||||||
|
@Parameter(description = "顺序翻页游标(上一页返回的 nextLastId;传了就忽略 page 偏移)")
|
||||||
|
@RequestParam(name = "last_id", required = false) Long lastId,
|
||||||
HttpServletRequest request) {
|
HttpServletRequest request) {
|
||||||
RequestOperator operator = requireDedupeTotalDataAccess(request);
|
RequestOperator operator = requireDedupeTotalDataAccess(request);
|
||||||
return ApiResponse.success(dedupeTotalDataService.page(
|
return ApiResponse.success(dedupeTotalDataService.page(
|
||||||
page, pageSize, keyword, username, startDate, endDate, groupId, country, operator.id()));
|
page, pageSize, keyword, username, startDate, endDate, groupId, country, lastId, operator.id()));
|
||||||
}
|
}
|
||||||
|
|
||||||
@GetMapping("/export")
|
@GetMapping("/export")
|
||||||
|
|||||||
+9
@@ -1,6 +1,8 @@
|
|||||||
package com.nanri.aiimage.modules.dedupe.model.entity;
|
package com.nanri.aiimage.modules.dedupe.model.entity;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.annotation.FieldStrategy;
|
||||||
import com.baomidou.mybatisplus.annotation.IdType;
|
import com.baomidou.mybatisplus.annotation.IdType;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableField;
|
||||||
import com.baomidou.mybatisplus.annotation.TableId;
|
import com.baomidou.mybatisplus.annotation.TableId;
|
||||||
import com.baomidou.mybatisplus.annotation.TableName;
|
import com.baomidou.mybatisplus.annotation.TableName;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -19,5 +21,12 @@ public class DedupeTotalDataEntity {
|
|||||||
private Long uploaderUserId;
|
private Long uploaderUserId;
|
||||||
private String uploaderUsername;
|
private String uploaderUsername;
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
|
/**
|
||||||
|
* 更新时间:由数据库维护(DEFAULT CURRENT_TIMESTAMP + ON UPDATE CURRENT_TIMESTAMP)。
|
||||||
|
*
|
||||||
|
* <p>禁止应用显式写:MySQL 在 UPDATE 语句显式给该列赋值时不会触发自动更新,
|
||||||
|
* 而本表写回为 selectById → 改字段 → updateById(实体带着旧值),一旦写回就会冻结更新时间。
|
||||||
|
*/
|
||||||
|
@TableField(value = "updated_at", insertStrategy = FieldStrategy.NEVER, updateStrategy = FieldStrategy.NEVER)
|
||||||
private LocalDateTime updatedAt;
|
private LocalDateTime updatedAt;
|
||||||
}
|
}
|
||||||
|
|||||||
+3
@@ -32,4 +32,7 @@ public class DedupeTotalDataItemVo {
|
|||||||
|
|
||||||
@Schema(description = "创建时间")
|
@Schema(description = "创建时间")
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
|
|
||||||
|
@Schema(description = "更新时间")
|
||||||
|
private LocalDateTime updatedAt;
|
||||||
}
|
}
|
||||||
|
|||||||
+5
@@ -20,4 +20,9 @@ public class DedupeTotalDataPageVo {
|
|||||||
|
|
||||||
@Schema(description = "每页数量")
|
@Schema(description = "每页数量")
|
||||||
private Long pageSize;
|
private Long pageSize;
|
||||||
|
/**
|
||||||
|
* 顺序翻页游标(2026-09 审查 C6):本页最后一行的 id;下一页把它回传为 last_id
|
||||||
|
* 即可走 keyset(索引顺序扫描 + LIMIT),避免深分页的 offset 代价。跳页仍用 page。
|
||||||
|
*/
|
||||||
|
private Long nextLastId;
|
||||||
}
|
}
|
||||||
|
|||||||
+82
-73
@@ -42,6 +42,7 @@ import java.util.concurrent.Executors;
|
|||||||
import java.util.concurrent.Future;
|
import java.util.concurrent.Future;
|
||||||
import java.util.concurrent.Semaphore;
|
import java.util.concurrent.Semaphore;
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
import java.util.function.Supplier;
|
||||||
import java.util.zip.ZipEntry;
|
import java.util.zip.ZipEntry;
|
||||||
import java.util.zip.ZipOutputStream;
|
import java.util.zip.ZipOutputStream;
|
||||||
|
|
||||||
@@ -487,7 +488,7 @@ public class DedupeRunService {
|
|||||||
readResult.scannedRows = new AtomicInteger(0);
|
readResult.scannedRows = new AtomicInteger(0);
|
||||||
readResult.filteredFbaRows = new AtomicInteger(0);
|
readResult.filteredFbaRows = new AtomicInteger(0);
|
||||||
|
|
||||||
PendingMainIdGroup pendingMainIdGroup = new PendingMainIdGroup();
|
IdRuleRowPicker rowPicker = new IdRuleRowPicker(keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds);
|
||||||
// 表头列索引缓存:由 onHeader 填充,数据行为空时使用
|
// 表头列索引缓存:由 onHeader 填充,数据行为空时使用
|
||||||
final Map<String, Integer>[] headerIndexCache = new Map[]{Map.of()};
|
final Map<String, Integer>[] headerIndexCache = new Map[]{Map.of()};
|
||||||
|
|
||||||
@@ -526,23 +527,16 @@ public class DedupeRunService {
|
|||||||
}
|
}
|
||||||
Integer idColumnIndex = headerIndex.get("id");
|
Integer idColumnIndex = headerIndex.get("id");
|
||||||
if (idColumnIndex == null) {
|
if (idColumnIndex == null) {
|
||||||
readResult.rows.add(buildCandidateRow(rowMap, headerIndex, orderedSelectedColumns, null));
|
rowPicker.addAlways(buildCandidateRow(rowMap, headerIndex, orderedSelectedColumns, null));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
appendRowByIdRule(
|
rowPicker.select(
|
||||||
normalizeCellText(cellText(rowMap, idColumnIndex)),
|
normalizeCellText(cellText(rowMap, idColumnIndex)),
|
||||||
rowMap,
|
() -> buildCandidateRow(rowMap, headerIndex, orderedSelectedColumns, null)
|
||||||
headerIndex,
|
|
||||||
orderedSelectedColumns,
|
|
||||||
keepIntegerIds,
|
|
||||||
keepUnderscoreIds,
|
|
||||||
keepIntegerMainIdsWhenNoSubIds,
|
|
||||||
pendingMainIdGroup,
|
|
||||||
readResult.rows
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
pendingMainIdGroup.flush(readResult.rows);
|
readResult.rows = rowPicker.resolve();
|
||||||
long readNs = elapsedNs(readStartNs);
|
long readNs = elapsedNs(readStartNs);
|
||||||
|
|
||||||
Set<String> candidateAsinValues = new HashSet<>();
|
Set<String> candidateAsinValues = new HashSet<>();
|
||||||
@@ -560,8 +554,9 @@ public class DedupeRunService {
|
|||||||
try (SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(200)) {
|
try (SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(200)) {
|
||||||
org.apache.poi.ss.usermodel.Sheet outputSheet = outputWorkbook.createSheet(WorkbookUtil.createSafeSheetName(readResult.sheetName.isBlank() ? "Sheet1" : readResult.sheetName));
|
org.apache.poi.ss.usermodel.Sheet outputSheet = outputWorkbook.createSheet(WorkbookUtil.createSafeSheetName(readResult.sheetName.isBlank() ? "Sheet1" : readResult.sheetName));
|
||||||
Row outputHeaderRow = outputSheet.createRow(0);
|
Row outputHeaderRow = outputSheet.createRow(0);
|
||||||
for (int i = 0; i < selectedColumns.size(); i++) {
|
// 表头必须与数据行同序:数据按 orderedSelectedColumns 取值,表头写 selectedColumns 会整体错列
|
||||||
outputHeaderRow.createCell(i).setCellValue(selectedColumns.get(i));
|
for (int i = 0; i < orderedSelectedColumns.size(); i++) {
|
||||||
|
outputHeaderRow.createCell(i).setCellValue(orderedSelectedColumns.get(i));
|
||||||
}
|
}
|
||||||
|
|
||||||
Set<String> writtenAsinValues = new HashSet<>();
|
Set<String> writtenAsinValues = new HashSet<>();
|
||||||
@@ -659,34 +654,80 @@ public class DedupeRunService {
|
|||||||
return new DedupeCandidateRow(selectedValues, asinValue);
|
return new DedupeCandidateRow(selectedValues, asinValue);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void appendRowByIdRule(String idValue, Map<Integer, String> rowMap,
|
/**
|
||||||
Map<String, Integer> headerIndex, List<String> selectedColumns,
|
* 按 ID 保留规则挑选输出行。
|
||||||
boolean keepIntegerIds, boolean keepUnderscoreIds,
|
*
|
||||||
boolean keepIntegerMainIdsWhenNoSubIds,
|
* <p>主链接行是否保留,只取决于「该主 ID 是否出现过子链接行」,与两类行在源文件中的
|
||||||
PendingMainIdGroup pendingMainIdGroup,
|
* 先后顺序无关。旧实现用「暂存主链接行 + 后到的同主 ID 子链接行把它丢弃」的方式,
|
||||||
List<DedupeCandidateRow> rows) {
|
* 一旦顺序是「子链接在前、主链接在后」(子行到来时暂存区尚空,无人记录该主 ID 已有
|
||||||
if (idValue == null || idValue.isBlank()) {
|
* 子链接),主链接就会一路存活到收尾,导致同一主 ID 的主/子链接同时出现在结果里。</p>
|
||||||
return;
|
*/
|
||||||
|
static final class IdRuleRowPicker {
|
||||||
|
|
||||||
|
private final boolean keepIntegerIds;
|
||||||
|
private final boolean keepUnderscoreIds;
|
||||||
|
private final boolean keepIntegerMainIdsWhenNoSubIds;
|
||||||
|
/** 出现过子链接行的主 ID 集合,与源文件顺序无关 */
|
||||||
|
private final Set<String> mainIdsWithSubRows = new HashSet<>();
|
||||||
|
private final List<PickedRow> pickedRows = new ArrayList<>();
|
||||||
|
private int subRowCount = 0;
|
||||||
|
private int mainRowCount = 0;
|
||||||
|
|
||||||
|
IdRuleRowPicker(boolean keepIntegerIds, boolean keepUnderscoreIds, boolean keepIntegerMainIdsWhenNoSubIds) {
|
||||||
|
this.keepIntegerIds = keepIntegerIds;
|
||||||
|
this.keepUnderscoreIds = keepUnderscoreIds;
|
||||||
|
this.keepIntegerMainIdsWhenNoSubIds = keepIntegerMainIdsWhenNoSubIds;
|
||||||
}
|
}
|
||||||
String mainId = extractMainId(idValue);
|
|
||||||
if (pendingMainIdGroup.hasDifferentMainId(mainId)) {
|
/** ID 不参与保留规则判定的行(如无 id 列或 id 为空)直接保留。 */
|
||||||
pendingMainIdGroup.flush(rows);
|
void addAlways(DedupeCandidateRow row) {
|
||||||
|
pickedRows.add(new PickedRow(row, null));
|
||||||
}
|
}
|
||||||
if (isUnderscoreId(idValue)) {
|
|
||||||
pendingMainIdGroup.discardIfSameMainId(mainId);
|
/** 按 ID 形态与保留规则挑选;rowSupplier 仅在确定保留时才求值。 */
|
||||||
if (keepUnderscoreIds) {
|
void select(String idValue, Supplier<DedupeCandidateRow> rowSupplier) {
|
||||||
rows.add(buildCandidateRow(rowMap, headerIndex, selectedColumns, null));
|
if (idValue == null || idValue.isBlank()) {
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (isIntegerId(idValue)) {
|
|
||||||
if (keepIntegerIds) {
|
|
||||||
rows.add(buildCandidateRow(rowMap, headerIndex, selectedColumns, null));
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (keepIntegerMainIdsWhenNoSubIds) {
|
String mainId = extractMainId(idValue);
|
||||||
pendingMainIdGroup.add(mainId, buildCandidateRow(rowMap, headerIndex, selectedColumns, null));
|
if (isUnderscoreId(idValue)) {
|
||||||
|
subRowCount++;
|
||||||
|
mainIdsWithSubRows.add(mainId);
|
||||||
|
if (keepUnderscoreIds) {
|
||||||
|
pickedRows.add(new PickedRow(rowSupplier.get(), null));
|
||||||
|
}
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
if (isIntegerId(idValue)) {
|
||||||
|
if (keepIntegerIds) {
|
||||||
|
pickedRows.add(new PickedRow(rowSupplier.get(), null));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (keepIntegerMainIdsWhenNoSubIds) {
|
||||||
|
mainRowCount++;
|
||||||
|
pickedRows.add(new PickedRow(rowSupplier.get(), mainId));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 收尾统一判定:带条件的主链接行仅在该主 ID 没有子链接行时保留。 */
|
||||||
|
List<DedupeCandidateRow> resolve() {
|
||||||
|
List<DedupeCandidateRow> resolved = new ArrayList<>(pickedRows.size());
|
||||||
|
int droppedMainRowCount = 0;
|
||||||
|
for (PickedRow picked : pickedRows) {
|
||||||
|
if (picked.conditionalMainId() != null && mainIdsWithSubRows.contains(picked.conditionalMainId())) {
|
||||||
|
droppedMainRowCount++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
resolved.add(picked.row());
|
||||||
|
}
|
||||||
|
log.info("dedupe id rules subRows={} mainRows={} mainIdsWithSubRows={} droppedMainRows={} keptRows={}",
|
||||||
|
subRowCount, mainRowCount, mainIdsWithSubRows.size(), droppedMainRowCount, resolved.size());
|
||||||
|
return resolved;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** conditionalMainId 非空表示该行是主链接行,需等收尾时看同主 ID 有无子链接行 */
|
||||||
|
private record PickedRow(DedupeCandidateRow row, String conditionalMainId) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -730,7 +771,7 @@ public class DedupeRunService {
|
|||||||
return String.join("/", parts);
|
return String.join("/", parts);
|
||||||
}
|
}
|
||||||
|
|
||||||
private String extractMainId(String text) {
|
static String extractMainId(String text) {
|
||||||
if (text == null || text.isBlank()) {
|
if (text == null || text.isBlank()) {
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
@@ -744,7 +785,7 @@ public class DedupeRunService {
|
|||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean isIntegerId(String text) {
|
static boolean isIntegerId(String text) {
|
||||||
if (text == null || text.isEmpty()) {
|
if (text == null || text.isEmpty()) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -756,7 +797,7 @@ public class DedupeRunService {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean isUnderscoreId(String text) {
|
static boolean isUnderscoreId(String text) {
|
||||||
if (text == null || text.length() < 3) {
|
if (text == null || text.length() < 3) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -952,38 +993,6 @@ public class DedupeRunService {
|
|||||||
private AtomicInteger filteredFbaRows;
|
private AtomicInteger filteredFbaRows;
|
||||||
}
|
}
|
||||||
|
|
||||||
private record DedupeCandidateRow(List<String> selectedValues, String asinValue) {
|
record DedupeCandidateRow(List<String> selectedValues, String asinValue) {
|
||||||
}
|
|
||||||
|
|
||||||
private static final class PendingMainIdGroup {
|
|
||||||
private String mainId;
|
|
||||||
private final List<DedupeCandidateRow> rows = new ArrayList<>();
|
|
||||||
|
|
||||||
private void add(String nextMainId, DedupeCandidateRow row) {
|
|
||||||
if (hasDifferentMainId(nextMainId)) {
|
|
||||||
rows.clear();
|
|
||||||
}
|
|
||||||
mainId = nextMainId;
|
|
||||||
rows.add(row);
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean hasDifferentMainId(String nextMainId) {
|
|
||||||
return mainId != null && (nextMainId == null || nextMainId.isBlank() || !mainId.equals(nextMainId));
|
|
||||||
}
|
|
||||||
|
|
||||||
private void discardIfSameMainId(String nextMainId) {
|
|
||||||
if (mainId != null && mainId.equals(nextMainId)) {
|
|
||||||
rows.clear();
|
|
||||||
mainId = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void flush(List<DedupeCandidateRow> outputRows) {
|
|
||||||
if (!rows.isEmpty()) {
|
|
||||||
outputRows.addAll(rows);
|
|
||||||
rows.clear();
|
|
||||||
}
|
|
||||||
mainId = null;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+20
-1
@@ -156,6 +156,16 @@ public class DedupeTotalDataService {
|
|||||||
|
|
||||||
public DedupeTotalDataPageVo page(long page, long pageSize, String keyword, String username,
|
public DedupeTotalDataPageVo page(long page, long pageSize, String keyword, String username,
|
||||||
LocalDate startDate, LocalDate endDate, Long groupId, String country, Long operatorId) {
|
LocalDate startDate, LocalDate endDate, Long groupId, String country, Long operatorId) {
|
||||||
|
return page(page, pageSize, keyword, username, startDate, endDate, groupId, country, null, operatorId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param lastId keyset 游标(上一页返回的 nextLastId);非空时忽略 page 偏移,
|
||||||
|
* 直接按 id 倒序取"比它小"的一页,避免深分页 offset 扫索引的代价
|
||||||
|
*/
|
||||||
|
public DedupeTotalDataPageVo page(long page, long pageSize, String keyword, String username,
|
||||||
|
LocalDate startDate, LocalDate endDate, Long groupId, String country,
|
||||||
|
Long lastId, Long operatorId) {
|
||||||
if (startDate != null && endDate != null && startDate.isAfter(endDate)) {
|
if (startDate != null && endDate != null && startDate.isAfter(endDate)) {
|
||||||
throw new BusinessException("开始日期不能晚于结束日期");
|
throw new BusinessException("开始日期不能晚于结束日期");
|
||||||
}
|
}
|
||||||
@@ -181,8 +191,14 @@ public class DedupeTotalDataService {
|
|||||||
.orderByDesc(DedupeTotalDataEntity::getId);
|
.orderByDesc(DedupeTotalDataEntity::getId);
|
||||||
applyGroupScope(query, scope, groupId);
|
applyGroupScope(query, scope, groupId);
|
||||||
Long total = dedupeTotalDataMapper.selectCount(query);
|
Long total = dedupeTotalDataMapper.selectCount(query);
|
||||||
|
boolean keyset = lastId != null && lastId > 0;
|
||||||
|
if (keyset) {
|
||||||
|
query.lt(DedupeTotalDataEntity::getId, lastId);
|
||||||
|
}
|
||||||
List<DedupeTotalDataEntity> rows = dedupeTotalDataMapper.selectList(
|
List<DedupeTotalDataEntity> rows = dedupeTotalDataMapper.selectList(
|
||||||
query.last("LIMIT " + ((safePage - 1) * safePageSize) + ", " + safePageSize));
|
keyset
|
||||||
|
? query.last("LIMIT " + safePageSize)
|
||||||
|
: query.last("LIMIT " + ((safePage - 1) * safePageSize) + ", " + safePageSize));
|
||||||
Map<Long, String> groupNames = loadGroupNames(rows);
|
Map<Long, String> groupNames = loadGroupNames(rows);
|
||||||
List<DedupeTotalDataItemVo> items = rows.stream()
|
List<DedupeTotalDataItemVo> items = rows.stream()
|
||||||
.map(row -> toItemVo(row, row.getGroupId() == null
|
.map(row -> toItemVo(row, row.getGroupId() == null
|
||||||
@@ -194,6 +210,8 @@ public class DedupeTotalDataService {
|
|||||||
vo.setTotal(total);
|
vo.setTotal(total);
|
||||||
vo.setPage(safePage);
|
vo.setPage(safePage);
|
||||||
vo.setPageSize(safePageSize);
|
vo.setPageSize(safePageSize);
|
||||||
|
// 顺序翻页游标:本页最后一行的 id(空页给 null,前端据此停止换页)
|
||||||
|
vo.setNextLastId(items.isEmpty() ? null : items.get(items.size() - 1).getId());
|
||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1380,6 +1398,7 @@ public class DedupeTotalDataService {
|
|||||||
vo.setUploaderUserId(entity.getUploaderUserId());
|
vo.setUploaderUserId(entity.getUploaderUserId());
|
||||||
vo.setUsername(entity.getUploaderUsername());
|
vo.setUsername(entity.getUploaderUsername());
|
||||||
vo.setCreatedAt(entity.getCreatedAt());
|
vo.setCreatedAt(entity.getCreatedAt());
|
||||||
|
vo.setUpdatedAt(entity.getUpdatedAt());
|
||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -102,8 +102,8 @@ public class DeleteBrandRunController {
|
|||||||
|
|
||||||
@PostMapping("/tasks/progress/light")
|
@PostMapping("/tasks/progress/light")
|
||||||
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
|
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
|
||||||
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt/rowsVersion),"
|
||||||
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
+ "不返回行明细内容(仅多一次结果行版本聚合查询),响应体更小;旧 progress/batch 端点保留不动。")
|
||||||
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||||
// 透传 userId:此前传 null = 不过滤归属,匿名遍历 taskId 即可读他人任务状态(2026-09 审查)
|
// 透传 userId:此前传 null = 不过滤归属,匿名遍历 taskId 即可读他人任务状态(2026-09 审查)
|
||||||
return ApiResponse.success(deleteBrandRunService.progressLight(request.getTaskIds(), request.getUserId()));
|
return ApiResponse.success(deleteBrandRunService.progressLight(request.getTaskIds(), request.getUserId()));
|
||||||
|
|||||||
+159
-1
@@ -21,9 +21,13 @@ import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService;
|
|||||||
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskCacheService;
|
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskCacheService;
|
||||||
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskService;
|
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskService;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskResumeService;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
@@ -55,6 +59,10 @@ public class DeleteBrandStaleTaskService {
|
|||||||
private static final String MODULE_TYPE_PATROL_DELETE = "PATROL_DELETE";
|
private static final String MODULE_TYPE_PATROL_DELETE = "PATROL_DELETE";
|
||||||
private static final String MODULE_TYPE_QUERY_ASIN = "QUERY_ASIN";
|
private static final String MODULE_TYPE_QUERY_ASIN = "QUERY_ASIN";
|
||||||
private static final String MODULE_TYPE_WITHDRAW = "WITHDRAW";
|
private static final String MODULE_TYPE_WITHDRAW = "WITHDRAW";
|
||||||
|
/** 二次判死线(心跳正常但无结果上报)覆盖的模块:走结果分片上报表、last_chunk_at 有效的模块。 */
|
||||||
|
private static final List<String> NO_RESULT_UPLOAD_CHECKED_MODULES = List.of(
|
||||||
|
MODULE_TYPE_DELETE_BRAND, MODULE_TYPE_PRODUCT_RISK, MODULE_TYPE_PRICE_TRACK,
|
||||||
|
MODULE_TYPE_SHOP_MATCH, MODULE_TYPE_PATROL_DELETE, MODULE_TYPE_QUERY_ASIN, MODULE_TYPE_WITHDRAW);
|
||||||
private static final Duration STALE_CHECK_LOCK_TTL = Duration.ofMinutes(10);
|
private static final Duration STALE_CHECK_LOCK_TTL = Duration.ofMinutes(10);
|
||||||
private static final Duration FINALIZE_CHECK_LOCK_TTL = Duration.ofMinutes(10);
|
private static final Duration FINALIZE_CHECK_LOCK_TTL = Duration.ofMinutes(10);
|
||||||
private static final Duration TEMP_DIR_CLEANUP_LOCK_TTL = Duration.ofMinutes(10);
|
private static final Duration TEMP_DIR_CLEANUP_LOCK_TTL = Duration.ofMinutes(10);
|
||||||
@@ -83,6 +91,10 @@ public class DeleteBrandStaleTaskService {
|
|||||||
private final TaskDistributedLockService taskDistributedLockService;
|
private final TaskDistributedLockService taskDistributedLockService;
|
||||||
private final TaskFileJobService taskFileJobService;
|
private final TaskFileJobService taskFileJobService;
|
||||||
private final ShopDataCrawlTaskService shopDataCrawlTaskService;
|
private final ShopDataCrawlTaskService shopDataCrawlTaskService;
|
||||||
|
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||||
|
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||||
|
/** 客户端中断任务的自动续跑(保留失败记录 + 重排队续跑任务,V129)。 */
|
||||||
|
private final TaskResumeService taskResumeService;
|
||||||
|
|
||||||
@Value("${aiimage.temp-dir.retention-hours:24}")
|
@Value("${aiimage.temp-dir.retention-hours:24}")
|
||||||
private long tempDirRetentionHours;
|
private long tempDirRetentionHours;
|
||||||
@@ -104,24 +116,45 @@ public class DeleteBrandStaleTaskService {
|
|||||||
ShopMatchStaleCheckStats patrolDeleteStats = failStalePatrolDeleteTasks();
|
ShopMatchStaleCheckStats patrolDeleteStats = failStalePatrolDeleteTasks();
|
||||||
ShopMatchStaleCheckStats queryAsinStats = failStaleQueryAsinTasks();
|
ShopMatchStaleCheckStats queryAsinStats = failStaleQueryAsinTasks();
|
||||||
ShopMatchStaleCheckStats withdrawStats = failStaleWithdrawTasks();
|
ShopMatchStaleCheckStats withdrawStats = failStaleWithdrawTasks();
|
||||||
|
NoResultUploadStaleCheckStats noUploadStats = failNoResultUploadTasks();
|
||||||
|
// 客户端重启中断的任务:除了标失败(客户端上报,用户能看到原因),还要重新排队
|
||||||
|
// 一条 PENDING 续跑任务交给客户端兜底拉取执行,否则长任务一遇客户端更新就整个白跑
|
||||||
|
TaskResumeService.ResumeStats resumeStats = resumeInterruptedSafely();
|
||||||
for (Map.Entry<String, Runnable> delegated : delegatedStaleChecks().entrySet()) {
|
for (Map.Entry<String, Runnable> delegated : delegatedStaleChecks().entrySet()) {
|
||||||
runModuleStaleCheck(delegated.getKey(), delegated.getValue());
|
runModuleStaleCheck(delegated.getKey(), delegated.getValue());
|
||||||
}
|
}
|
||||||
// 周期每 2 分钟一轮,各模块 summary 合并为单行,避免定期刷屏
|
// 周期每 2 分钟一轮,各模块 summary 合并为单行,避免定期刷屏
|
||||||
// 注意:每段占位符数量必须与实参一致——此前每段 5 个占位符只传 4 个参数,
|
// 注意:每段占位符数量必须与实参一致——此前每段 5 个占位符只传 4 个参数,
|
||||||
// 导致 withdraw 之后的取值整体错位、末尾 elapsedMs/thread 打成字面量
|
// 导致 withdraw 之后的取值整体错位、末尾 elapsedMs/thread 打成字面量
|
||||||
log.info("[stale-check] summary product-risk(s={} f={} x={} p={}) price-track(s={} f={} x={} p={}) shop-match(s={} f={} x={} p={}) patrol-delete(s={} f={} x={} p={}) query-asin(s={} f={} x={} p={}) withdraw(s={} f={} x={} p={}) elapsedMs={} thread={}",
|
log.info("[stale-check] summary product-risk(s={} f={} x={} p={}) price-track(s={} f={} x={} p={}) shop-match(s={} f={} x={} p={}) patrol-delete(s={} f={} x={} p={}) query-asin(s={} f={} x={} p={}) withdraw(s={} f={} x={} p={}) no-upload(c={} f={} x={}) resume(s={} r={} k={} u={}) elapsedMs={} thread={}",
|
||||||
stats.scannedTaskCount, stats.finalizedTaskCount, stats.failedTaskCount, stats.skippedTaskCount,
|
stats.scannedTaskCount, stats.finalizedTaskCount, stats.failedTaskCount, stats.skippedTaskCount,
|
||||||
priceTrackStats.scannedTaskCount, priceTrackStats.finalizedTaskCount, priceTrackStats.failedTaskCount, priceTrackStats.skippedTaskCount,
|
priceTrackStats.scannedTaskCount, priceTrackStats.finalizedTaskCount, priceTrackStats.failedTaskCount, priceTrackStats.skippedTaskCount,
|
||||||
shopMatchStats.scannedTaskCount, shopMatchStats.finalizedTaskCount, shopMatchStats.failedTaskCount, shopMatchStats.skippedTaskCount,
|
shopMatchStats.scannedTaskCount, shopMatchStats.finalizedTaskCount, shopMatchStats.failedTaskCount, shopMatchStats.skippedTaskCount,
|
||||||
patrolDeleteStats.scannedTaskCount, patrolDeleteStats.finalizedTaskCount, patrolDeleteStats.failedTaskCount, patrolDeleteStats.skippedTaskCount,
|
patrolDeleteStats.scannedTaskCount, patrolDeleteStats.finalizedTaskCount, patrolDeleteStats.failedTaskCount, patrolDeleteStats.skippedTaskCount,
|
||||||
queryAsinStats.scannedTaskCount, queryAsinStats.finalizedTaskCount, queryAsinStats.failedTaskCount, queryAsinStats.skippedTaskCount,
|
queryAsinStats.scannedTaskCount, queryAsinStats.finalizedTaskCount, queryAsinStats.failedTaskCount, queryAsinStats.skippedTaskCount,
|
||||||
withdrawStats.scannedTaskCount, withdrawStats.finalizedTaskCount, withdrawStats.failedTaskCount, withdrawStats.skippedTaskCount,
|
withdrawStats.scannedTaskCount, withdrawStats.finalizedTaskCount, withdrawStats.failedTaskCount, withdrawStats.skippedTaskCount,
|
||||||
|
noUploadStats.scannedTaskCount, noUploadStats.failedTaskCount, noUploadStats.skippedTaskCount,
|
||||||
|
resumeStats.scannedTaskCount, resumeStats.resumedTaskCount, resumeStats.skippedTaskCount,
|
||||||
|
resumeStats.unsupportedTaskCount,
|
||||||
System.currentTimeMillis() - startedAt,
|
System.currentTimeMillis() - startedAt,
|
||||||
Thread.currentThread().getName());
|
Thread.currentThread().getName());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 自动续跑巡检的隔离壳:续跑失败绝不能拖垮判死主流程。
|
||||||
|
* 判死是任务状态的兜底(不做会留下永远 RUNNING 的孤儿),优先级高于续跑;
|
||||||
|
* 这里失败只记日志并返回空统计,2 分钟后的下一轮自然重试。
|
||||||
|
*/
|
||||||
|
private TaskResumeService.ResumeStats resumeInterruptedSafely() {
|
||||||
|
try {
|
||||||
|
return taskResumeService.resumeInterruptedTasks();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[task-resume] 自动续跑巡检失败(不影响本轮判死): {}", ex.getMessage(), ex);
|
||||||
|
return new TaskResumeService.ResumeStats();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 委派式陈旧判死:moduleType → 处理动作。
|
* 委派式陈旧判死:moduleType → 处理动作。
|
||||||
* {@code TaskModuleRegistry} 中 delegatedStaleCheck=true 的模块都必须在这里登记,
|
* {@code TaskModuleRegistry} 中 delegatedStaleCheck=true 的模块都必须在这里登记,
|
||||||
@@ -711,6 +744,125 @@ public class DeleteBrandStaleTaskService {
|
|||||||
return stats;
|
return stats;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 二次判死:心跳正常但连续 N 分钟无结果分片上报(治「心跳续命」的假活任务)。
|
||||||
|
*
|
||||||
|
* <p>既有各模块心跳线的候选条件都是 updated_at 陈旧,而客户端心跳会持续刷新 updated_at——
|
||||||
|
* 主线程卡死(浏览器自动化等待/异常)时心跳线程照发,任务永远命不中(生产 28131 卡死 12h+ 仍 RUNNING)。
|
||||||
|
* 本线改用 biz_task_scope_state.last_chunk_at(只随结果分片上报刷新)作判据;
|
||||||
|
* 无 scope 行或 last_chunk_at 全 NULL(从未上报)的任务跳过,避免误杀首批较慢的正常任务。
|
||||||
|
*/
|
||||||
|
private NoResultUploadStaleCheckStats failNoResultUploadTasks() {
|
||||||
|
NoResultUploadStaleCheckStats stats = new NoResultUploadStaleCheckStats();
|
||||||
|
if (!deleteBrandProgressProperties.isNoResultUploadCheckEnabled()) {
|
||||||
|
return stats;
|
||||||
|
}
|
||||||
|
long minutes = Math.max(1L, deleteBrandProgressProperties.getNoResultUploadTimeoutMinutes());
|
||||||
|
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(minutes);
|
||||||
|
List<FileTaskEntity> candidates = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
|
.select(FileTaskEntity::getId, FileTaskEntity::getModuleType,
|
||||||
|
FileTaskEntity::getCreatedAt, FileTaskEntity::getUpdatedAt)
|
||||||
|
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||||
|
.in(FileTaskEntity::getModuleType, NO_RESULT_UPLOAD_CHECKED_MODULES)
|
||||||
|
.lt(FileTaskEntity::getCreatedAt, cutoff)
|
||||||
|
.orderByAsc(FileTaskEntity::getCreatedAt)
|
||||||
|
.last("limit 200"));
|
||||||
|
stats.scannedTaskCount = candidates.size();
|
||||||
|
if (candidates.isEmpty()) {
|
||||||
|
return stats;
|
||||||
|
}
|
||||||
|
Map<Long, LocalDateTime> lastResultAtByTaskId = taskScopeStateMapper
|
||||||
|
.selectLastChunkAtByTaskIds(candidates.stream().map(FileTaskEntity::getId).toList())
|
||||||
|
.stream()
|
||||||
|
.filter(dto -> dto.taskId() != null && dto.lastChunkAt() != null)
|
||||||
|
.collect(Collectors.toMap(TaskScopeLastChunkDto::taskId, TaskScopeLastChunkDto::lastChunkAt, (a, b) -> a));
|
||||||
|
for (FileTaskEntity task : candidates) {
|
||||||
|
LocalDateTime lastResultAt = lastResultAtByTaskId.get(task.getId());
|
||||||
|
// 从未上报(无 scope 行或全 NULL)或上报仍新鲜:正常推进,不必处理
|
||||||
|
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
String moduleType = task.getModuleType();
|
||||||
|
if (hasPendingAssembleJobs(task.getId(), moduleType)) {
|
||||||
|
stats.skippedTaskCount++;
|
||||||
|
log.info("[stale-check] no-upload skip pending-assemble-jobs taskId={} moduleType={}", task.getId(), moduleType);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(moduleType, task.getId());
|
||||||
|
if (taskLockHandle == null) {
|
||||||
|
stats.skippedTaskCount++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try (taskLockHandle) {
|
||||||
|
try {
|
||||||
|
tryFinalizeNoResultUploadTask(moduleType, task.getId());
|
||||||
|
} catch (Exception ex) {
|
||||||
|
// finalize 抛异常(组装失败等)不代表任务活着:继续走 CAS 判死
|
||||||
|
log.warn("[stale-check] no-upload finalize threw taskId={} moduleType={} msg={}", task.getId(), moduleType, ex.getMessage());
|
||||||
|
}
|
||||||
|
// finalize 已终结的任务 CAS 自然不命中(状态不再是 RUNNING),无需回读
|
||||||
|
String lastPosition = taskHeartbeatPositionService.describe(task.getId());
|
||||||
|
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||||
|
.eq(FileTaskEntity::getId, task.getId())
|
||||||
|
.eq(FileTaskEntity::getModuleType, moduleType)
|
||||||
|
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||||
|
.set(FileTaskEntity::getStatus, "FAILED")
|
||||||
|
.set(FileTaskEntity::getErrorMessage, buildNoResultUploadFailReason(minutes, lastResultAt, lastPosition))
|
||||||
|
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||||
|
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||||
|
if (updated > 0) {
|
||||||
|
stats.failedTaskCount++;
|
||||||
|
deleteNoResultUploadTaskCache(moduleType, task.getId());
|
||||||
|
log.warn("[stale-check] no-upload failed taskId={} moduleType={} lastResultAt={} taskUpdatedAt={} timeoutMinutes={}",
|
||||||
|
task.getId(), moduleType, lastResultAt, task.getUpdatedAt(), minutes);
|
||||||
|
} else {
|
||||||
|
stats.skippedTaskCount++;
|
||||||
|
log.info("[stale-check] no-upload task already finalized by compensation taskId={} moduleType={}", task.getId(), moduleType);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return stats;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 按模块调用各自的收尾入口(尽力组装部分结果;已终结的任务由后续 CAS 自然放行)。 */
|
||||||
|
private void tryFinalizeNoResultUploadTask(String moduleType, Long taskId) {
|
||||||
|
switch (moduleType) {
|
||||||
|
case MODULE_TYPE_DELETE_BRAND -> deleteBrandRunService.tryFinalizeTask(taskId, true);
|
||||||
|
case MODULE_TYPE_PRODUCT_RISK -> productRiskTaskService.tryFinalizeTask(taskId, true);
|
||||||
|
case MODULE_TYPE_PRICE_TRACK -> priceTrackTaskService.tryFinalizeTask(taskId, true);
|
||||||
|
case MODULE_TYPE_SHOP_MATCH -> shopMatchTaskService.tryFinalizeTask(taskId, true);
|
||||||
|
case MODULE_TYPE_PATROL_DELETE -> patrolDeleteTaskService.tryFinalizeTask(taskId, true);
|
||||||
|
case MODULE_TYPE_QUERY_ASIN -> queryAsinTaskService.tryFinalizeTask(taskId, true);
|
||||||
|
case MODULE_TYPE_WITHDRAW -> withdrawTaskService.tryFinalizeTask(taskId, true);
|
||||||
|
default -> { }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 按模块删除各自缓存(与既有各段 CAS 翻转后的清理一致)。 */
|
||||||
|
private void deleteNoResultUploadTaskCache(String moduleType, Long taskId) {
|
||||||
|
switch (moduleType) {
|
||||||
|
case MODULE_TYPE_DELETE_BRAND -> deleteBrandTaskCacheService.delete(taskId);
|
||||||
|
case MODULE_TYPE_PRODUCT_RISK -> productRiskTaskCacheService.deleteTaskCache(taskId);
|
||||||
|
case MODULE_TYPE_PRICE_TRACK -> priceTrackTaskCacheService.deleteTaskCache(taskId);
|
||||||
|
case MODULE_TYPE_SHOP_MATCH -> shopMatchTaskCacheService.deleteTaskCache(taskId);
|
||||||
|
case MODULE_TYPE_PATROL_DELETE -> patrolDeleteTaskCacheService.deleteTaskCache(taskId);
|
||||||
|
case MODULE_TYPE_QUERY_ASIN -> queryAsinTaskCacheService.deleteTaskCache(taskId);
|
||||||
|
case MODULE_TYPE_WITHDRAW -> withdrawTaskCacheService.deleteTaskCache(taskId);
|
||||||
|
default -> { }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 二次判死文案:写明最后结果上报时间与最后处理位置,便于区分「卡死」与「长间隔」并定位卡点。 */
|
||||||
|
private String buildNoResultUploadFailReason(long minutes, LocalDateTime lastResultAt, String position) {
|
||||||
|
StringBuilder sb = new StringBuilder("连续 ").append(minutes)
|
||||||
|
.append(" 分钟无结果回传,疑似客户端任务卡死,任务已自动失败(最后结果上报 ").append(lastResultAt);
|
||||||
|
if (position != null && !position.isBlank()) {
|
||||||
|
sb.append(",最后处理位置:").append(position);
|
||||||
|
}
|
||||||
|
sb.append(")");
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
private TaskDistributedLockService.LockHandle acquireTaskLock(String moduleType, Long taskId) {
|
private TaskDistributedLockService.LockHandle acquireTaskLock(String moduleType, Long taskId) {
|
||||||
TaskDistributedLockService.LockHandle lockHandle = taskDistributedLockService.acquire(moduleType, taskId, 0L);
|
TaskDistributedLockService.LockHandle lockHandle = taskDistributedLockService.acquire(moduleType, taskId, 0L);
|
||||||
if (lockHandle == null) {
|
if (lockHandle == null) {
|
||||||
@@ -849,4 +1001,10 @@ public class DeleteBrandStaleTaskService {
|
|||||||
private int failedTaskCount;
|
private int failedTaskCount;
|
||||||
private int skippedTaskCount;
|
private int skippedTaskCount;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static final class NoResultUploadStaleCheckStats {
|
||||||
|
private int scannedTaskCount;
|
||||||
|
private int failedTaskCount;
|
||||||
|
private int skippedTaskCount;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+61
-11
@@ -16,6 +16,8 @@ import lombok.extern.slf4j.Slf4j;
|
|||||||
import org.springframework.dao.DuplicateKeyException;
|
import org.springframework.dao.DuplicateKeyException;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
import org.springframework.transaction.support.TransactionSynchronization;
|
||||||
|
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||||
|
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.security.MessageDigest;
|
import java.security.MessageDigest;
|
||||||
@@ -252,6 +254,13 @@ public class DeleteBrandTaskStorageService {
|
|||||||
return grouped;
|
return grouped;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 删除任务的全部范围/分片数据。
|
||||||
|
*
|
||||||
|
* <p>事务边界:远端载荷删除移到事务提交后执行 —— 单次删除超时 120s、重试 3 次,
|
||||||
|
* 放在 DB 事务里会让连接被长时间占用(项目在 TaskScopePayloadStorageService 已写明该约束)。
|
||||||
|
* 提交后再删还有个好处:引用计数查询看到的是删除完成后的 DB 状态,判断更准确。
|
||||||
|
*/
|
||||||
@Transactional
|
@Transactional
|
||||||
public void deleteTaskData(Long taskId) {
|
public void deleteTaskData(Long taskId) {
|
||||||
if (taskId == null || taskId <= 0) {
|
if (taskId == null || taskId <= 0) {
|
||||||
@@ -261,27 +270,68 @@ public class DeleteBrandTaskStorageService {
|
|||||||
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
|
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
|
||||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
||||||
if (states != null) {
|
|
||||||
for (TaskScopeStateEntity state : states) {
|
|
||||||
transientPayloadStorageService.deletePayloadIfPresent(state.getParsedPayloadJson());
|
|
||||||
transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
.select(TaskChunkEntity::getPayloadJson)
|
.select(TaskChunkEntity::getPayloadJson)
|
||||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||||
if (chunks != null) {
|
|
||||||
for (TaskChunkEntity chunk : chunks) {
|
|
||||||
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
|
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||||
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
||||||
|
deletePayloadsAfterCommit(states, chunks, taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 事务提交后逐条删远端载荷;无事务时立即执行(与 TaskResultItemService 同款兜底)。 */
|
||||||
|
private void deletePayloadsAfterCommit(List<TaskScopeStateEntity> states,
|
||||||
|
List<TaskChunkEntity> chunks,
|
||||||
|
Long taskId) {
|
||||||
|
List<String> payloads = new ArrayList<>();
|
||||||
|
if (states != null) {
|
||||||
|
for (TaskScopeStateEntity state : states) {
|
||||||
|
if (state == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (state.getParsedPayloadJson() != null && !state.getParsedPayloadJson().isBlank()) {
|
||||||
|
payloads.add(state.getParsedPayloadJson());
|
||||||
|
}
|
||||||
|
if (state.getStateJson() != null && !state.getStateJson().isBlank()) {
|
||||||
|
payloads.add(state.getStateJson());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (chunks != null) {
|
||||||
|
for (TaskChunkEntity chunk : chunks) {
|
||||||
|
if (chunk != null && chunk.getPayloadJson() != null && !chunk.getPayloadJson().isBlank()) {
|
||||||
|
payloads.add(chunk.getPayloadJson());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (payloads.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (TransactionSynchronizationManager.isSynchronizationActive()) {
|
||||||
|
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
|
||||||
|
@Override
|
||||||
|
public void afterCommit() {
|
||||||
|
deletePayloadsNow(payloads, taskId);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
deletePayloadsNow(payloads, taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void deletePayloadsNow(List<String> payloads, Long taskId) {
|
||||||
|
for (String payload : payloads) {
|
||||||
|
try {
|
||||||
|
transientPayloadStorageService.deletePayloadIfPresent(payload);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
// 事务已提交,远端删除失败只记日志,不影响任务数据清理结果
|
||||||
|
log.warn("[delete-brand-storage] 载荷删除失败 taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void refreshScopeState(Long taskId, String scopeKey, String scopeHash, Integer chunkTotal, LocalDateTime now) {
|
private void refreshScopeState(Long taskId, String scopeKey, String scopeHash, Integer chunkTotal, LocalDateTime now) {
|
||||||
|
|||||||
+159
@@ -0,0 +1,159 @@
|
|||||||
|
package com.nanri.aiimage.modules.devicelog.controller;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.api.ApiResponse;
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.common.model.entity.AdminUserEntity;
|
||||||
|
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||||
|
import com.nanri.aiimage.modules.devicelog.model.entity.DeviceLogConfigEntity;
|
||||||
|
import com.nanri.aiimage.modules.devicelog.model.entity.DeviceLogFileEntity;
|
||||||
|
import com.nanri.aiimage.modules.devicelog.model.vo.DeviceLogContentVo;
|
||||||
|
import com.nanri.aiimage.modules.devicelog.model.vo.DeviceLogPageVo;
|
||||||
|
import com.nanri.aiimage.modules.devicelog.service.DeviceLogConfigService;
|
||||||
|
import com.nanri.aiimage.modules.devicelog.service.DeviceLogService;
|
||||||
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
|
import io.swagger.v3.oas.annotations.Parameter;
|
||||||
|
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||||
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
|
import jakarta.servlet.http.HttpServletResponse;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.format.annotation.DateTimeFormat;
|
||||||
|
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||||
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
|
import org.springframework.web.bind.annotation.PathVariable;
|
||||||
|
import org.springframework.web.bind.annotation.PutMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestParam;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.URLEncoder;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.time.LocalDate;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 后台「日志管理」:桌面客户端与麦象采集机的日志浏览(仅超管)。
|
||||||
|
* 列表/内容/下载/删除 + 采集配置(全局默认与终端覆盖)。
|
||||||
|
*/
|
||||||
|
@RestController
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
@RequestMapping("/api/admin/device-logs")
|
||||||
|
@Tag(name = "日志管理(后台)", description = "设备日志列表、内容查看、下载与采集配置(仅超管)。")
|
||||||
|
public class AdminDeviceLogController {
|
||||||
|
|
||||||
|
private final DeviceLogService deviceLogService;
|
||||||
|
private final DeviceLogConfigService deviceLogConfigService;
|
||||||
|
private final AdminAuthSupport adminAuthSupport;
|
||||||
|
|
||||||
|
@GetMapping("/files")
|
||||||
|
@Operation(summary = "日志文件分页列表",
|
||||||
|
description = "source=client/maixiang;keyword 模糊匹配设备名/设备ID/文件名;日期为闭区间(早于保留窗口自动收紧)。")
|
||||||
|
public ApiResponse<DeviceLogPageVo> files(
|
||||||
|
HttpServletRequest request,
|
||||||
|
@Parameter(description = "来源") @RequestParam(required = false) String source,
|
||||||
|
@Parameter(description = "关键字(设备/文件名)") @RequestParam(required = false) String keyword,
|
||||||
|
@Parameter(description = "起始日期(含)") @RequestParam(required = false)
|
||||||
|
@DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate startDate,
|
||||||
|
@Parameter(description = "结束日期(含)") @RequestParam(required = false)
|
||||||
|
@DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate,
|
||||||
|
@RequestParam(defaultValue = "1") Long page,
|
||||||
|
@RequestParam(defaultValue = "20") Long pageSize) {
|
||||||
|
requireSuperAdmin(request);
|
||||||
|
return ApiResponse.success(deviceLogService.page(source, keyword, startDate, endDate, page, pageSize));
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/content")
|
||||||
|
@Operation(summary = "查看日志尾部内容", description = "默认取最后 256KB(自最早行边界起);truncated=true 时可用更大 maxBytes 再取。")
|
||||||
|
public ApiResponse<DeviceLogContentVo> content(
|
||||||
|
HttpServletRequest request,
|
||||||
|
@RequestParam Long fileId,
|
||||||
|
@Parameter(description = "期望返回的明文字节数(16KB ~ 8MB)") @RequestParam(required = false) Long maxBytes) {
|
||||||
|
requireSuperAdmin(request);
|
||||||
|
return ApiResponse.success(deviceLogService.readTail(fileId, maxBytes));
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/download")
|
||||||
|
@Operation(summary = "下载完整日志(按偏移拼接解压)")
|
||||||
|
public void download(HttpServletRequest request, HttpServletResponse response,
|
||||||
|
@RequestParam Long fileId) throws IOException {
|
||||||
|
requireSuperAdmin(request);
|
||||||
|
DeviceLogFileEntity row = deviceLogService.requireFile(fileId);
|
||||||
|
String downloadName = row.getFileName().replace('/', '_').replace('\\', '_');
|
||||||
|
response.setContentType("text/plain;charset=UTF-8");
|
||||||
|
response.setHeader("Content-Disposition", "attachment; filename*=UTF-8''"
|
||||||
|
+ URLEncoder.encode(downloadName, StandardCharsets.UTF_8).replace("+", "%20"));
|
||||||
|
deviceLogService.streamDownload(fileId, response.getOutputStream());
|
||||||
|
}
|
||||||
|
|
||||||
|
@DeleteMapping("/{id}")
|
||||||
|
@Operation(summary = "删除日志文件(片段与元数据,不可恢复)")
|
||||||
|
public ApiResponse<Map<String, Object>> delete(HttpServletRequest request,
|
||||||
|
@PathVariable Long id) {
|
||||||
|
requireSuperAdmin(request);
|
||||||
|
int deletedParts = deviceLogService.deleteFile(id);
|
||||||
|
return ApiResponse.success("已删除", Map.of("deletedParts", deletedParts));
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/config")
|
||||||
|
@Operation(summary = "采集配置:全局默认 + 终端覆盖列表")
|
||||||
|
public ApiResponse<Map<String, Object>> config(HttpServletRequest request,
|
||||||
|
@RequestParam(required = false) String keyword) {
|
||||||
|
requireSuperAdmin(request);
|
||||||
|
Map<String, Object> data = new LinkedHashMap<>();
|
||||||
|
data.put("globalMode", deviceLogConfigService.globalMode());
|
||||||
|
List<DeviceLogConfigEntity> overrides = deviceLogConfigService.listOverrides(keyword);
|
||||||
|
data.put("overrides", overrides);
|
||||||
|
return ApiResponse.success(data);
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/devices")
|
||||||
|
@Operation(summary = "最近上报的终端列表(覆盖选择用)")
|
||||||
|
public ApiResponse<List<Map<String, Object>>> devices(HttpServletRequest request) {
|
||||||
|
requireSuperAdmin(request);
|
||||||
|
return ApiResponse.success(deviceLogService.recentDevices());
|
||||||
|
}
|
||||||
|
|
||||||
|
@PutMapping("/config/global")
|
||||||
|
@Operation(summary = "设置全局采集模式", description = "mode=full(全量)/ selected(精选)")
|
||||||
|
public ApiResponse<Map<String, Object>> updateGlobal(HttpServletRequest request,
|
||||||
|
@RequestParam String mode) {
|
||||||
|
requireSuperAdmin(request);
|
||||||
|
deviceLogConfigService.setGlobalMode(mode);
|
||||||
|
return ApiResponse.success(Map.of("globalMode", deviceLogConfigService.globalMode()));
|
||||||
|
}
|
||||||
|
|
||||||
|
@PutMapping("/config/device")
|
||||||
|
@Operation(summary = "设置/更新终端采集模式覆盖")
|
||||||
|
public ApiResponse<Map<String, Object>> updateDevice(HttpServletRequest request,
|
||||||
|
@RequestParam String source,
|
||||||
|
@RequestParam String deviceId,
|
||||||
|
@RequestParam(required = false) String deviceName,
|
||||||
|
@RequestParam String mode) {
|
||||||
|
requireSuperAdmin(request);
|
||||||
|
DeviceLogConfigEntity row = deviceLogConfigService.upsertOverride(source, deviceId, deviceName, mode);
|
||||||
|
return ApiResponse.success(Map.of("id", row.getId()));
|
||||||
|
}
|
||||||
|
|
||||||
|
@DeleteMapping("/config/device/{id}")
|
||||||
|
@Operation(summary = "删除终端覆盖(回落到全局默认)")
|
||||||
|
public ApiResponse<Boolean> deleteOverride(HttpServletRequest request, @PathVariable Long id) {
|
||||||
|
requireSuperAdmin(request);
|
||||||
|
return ApiResponse.success("已删除", deviceLogConfigService.deleteOverride(id));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 日志可能含账号/代理等敏感信息,这里比常规后台更严:仅超管(requireAdmin 不含)。
|
||||||
|
*/
|
||||||
|
private void requireSuperAdmin(HttpServletRequest request) {
|
||||||
|
AdminUserEntity user = adminAuthSupport.requireAdmin(request);
|
||||||
|
if (!"super_admin".equals(adminAuthSupport.currentRole(user))) {
|
||||||
|
log.warn("[device-log] 非超管访问日志管理被拒 userId={} username={}",
|
||||||
|
user.getId(), user.getUsername());
|
||||||
|
throw new BusinessException(403, "仅超级管理员可访问日志管理");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+102
@@ -0,0 +1,102 @@
|
|||||||
|
package com.nanri.aiimage.modules.devicelog.controller;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.api.ApiResponse;
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||||
|
import com.nanri.aiimage.modules.devicelog.service.DeviceLogConfigService;
|
||||||
|
import com.nanri.aiimage.modules.devicelog.service.DeviceLogService;
|
||||||
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
|
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||||
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.format.annotation.DateTimeFormat;
|
||||||
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestParam;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.time.LocalDate;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 设备日志上报(桌面客户端 / 麦象采集机):增量片段上传、进度对齐、采集配置拉取。
|
||||||
|
* 仅内部令牌(X-Internal-Token)可调。
|
||||||
|
*/
|
||||||
|
@RestController
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
@RequestMapping("/api/internal/device-logs")
|
||||||
|
@Tag(name = "内部接口", description = "服务间调用(X-Internal-Token 鉴权)。")
|
||||||
|
public class InternalDeviceLogController {
|
||||||
|
|
||||||
|
private final DeviceLogService deviceLogService;
|
||||||
|
private final DeviceLogConfigService deviceLogConfigService;
|
||||||
|
private final AdminAuthSupport adminAuthSupport;
|
||||||
|
|
||||||
|
@PostMapping("/upload")
|
||||||
|
@Operation(summary = "上报日志增量片段",
|
||||||
|
description = "multipart:元数据字段 + file(gzip 片段)。offset 必须等于服务端已收字节数;"
|
||||||
|
+ "重复片段幂等跳过(skipped=true);偏移不连续返回 code=409,调用方应从 uploadedBytes 重读。")
|
||||||
|
public ApiResponse<Map<String, Object>> upload(
|
||||||
|
HttpServletRequest request,
|
||||||
|
@RequestParam("source") String source,
|
||||||
|
@RequestParam("deviceId") String deviceId,
|
||||||
|
@RequestParam(value = "deviceName", required = false) String deviceName,
|
||||||
|
@RequestParam(value = "uid", required = false) Long uid,
|
||||||
|
@RequestParam("fileName") String fileName,
|
||||||
|
@RequestParam("logDate") @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate logDate,
|
||||||
|
@RequestParam("offset") long offset,
|
||||||
|
@RequestParam("plainBytes") long plainBytes,
|
||||||
|
@RequestParam("file") MultipartFile file) throws IOException {
|
||||||
|
requireInternal(request, "日志上报");
|
||||||
|
if (file == null || file.isEmpty()) {
|
||||||
|
return ApiResponse.fail("file 片段为空");
|
||||||
|
}
|
||||||
|
DeviceLogService.PartResult result = deviceLogService.recordPart(source, deviceId, deviceName, uid,
|
||||||
|
fileName, logDate, offset, plainBytes, file.getBytes());
|
||||||
|
Map<String, Object> data = new LinkedHashMap<>();
|
||||||
|
data.put("uploadedBytes", result.uploadedBytes());
|
||||||
|
data.put("partCount", result.partCount());
|
||||||
|
data.put("accepted", result.accepted());
|
||||||
|
data.put("skipped", result.skipped());
|
||||||
|
return ApiResponse.success(data);
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/state")
|
||||||
|
@Operation(summary = "查询某文件服务端已收进度", description = "客户端本地进度丢失/被拒后从此对齐。")
|
||||||
|
public ApiResponse<Map<String, Object>> state(
|
||||||
|
HttpServletRequest request,
|
||||||
|
@RequestParam("source") String source,
|
||||||
|
@RequestParam("deviceId") String deviceId,
|
||||||
|
@RequestParam("fileName") String fileName,
|
||||||
|
@RequestParam("logDate") @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate logDate) {
|
||||||
|
requireInternal(request, "进度查询");
|
||||||
|
return ApiResponse.success(deviceLogService.state(source, deviceId, fileName, logDate));
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/config")
|
||||||
|
@Operation(summary = "拉取生效的采集配置", description = "终端覆盖 > 全局默认;返回 mode 与精选模式排除清单(glob)。")
|
||||||
|
public ApiResponse<Map<String, Object>> config(
|
||||||
|
HttpServletRequest request,
|
||||||
|
@RequestParam("source") String source,
|
||||||
|
@RequestParam("deviceId") String deviceId) {
|
||||||
|
requireInternal(request, "配置拉取");
|
||||||
|
DeviceLogConfigService.EffectiveConfig config = deviceLogConfigService.resolve(source, deviceId);
|
||||||
|
Map<String, Object> data = new LinkedHashMap<>();
|
||||||
|
data.put("mode", config.mode());
|
||||||
|
data.put("exclude", config.exclude());
|
||||||
|
return ApiResponse.success(data);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void requireInternal(HttpServletRequest request, String scene) {
|
||||||
|
if (!adminAuthSupport.isTrustedInternalToken(request)) {
|
||||||
|
log.warn("[device-log] 拒绝未携带可信内部令牌的{}请求 remoteAddr={}", scene, request.getRemoteAddr());
|
||||||
|
throw new BusinessException(401, "未授权");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+9
@@ -0,0 +1,9 @@
|
|||||||
|
package com.nanri.aiimage.modules.devicelog.mapper;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||||
|
import com.nanri.aiimage.modules.devicelog.model.entity.DeviceLogConfigEntity;
|
||||||
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
|
|
||||||
|
@Mapper
|
||||||
|
public interface DeviceLogConfigMapper extends BaseMapper<DeviceLogConfigEntity> {
|
||||||
|
}
|
||||||
+26
@@ -0,0 +1,26 @@
|
|||||||
|
package com.nanri.aiimage.modules.devicelog.mapper;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||||
|
import com.nanri.aiimage.modules.devicelog.model.entity.DeviceLogFileEntity;
|
||||||
|
import org.apache.ibatis.annotations.Delete;
|
||||||
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
|
import org.apache.ibatis.annotations.Param;
|
||||||
|
|
||||||
|
import java.time.LocalDate;
|
||||||
|
|
||||||
|
@Mapper
|
||||||
|
public interface DeviceLogFileMapper extends BaseMapper<DeviceLogFileEntity> {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 按日志日期分批删除保留期外的元数据行。
|
||||||
|
*
|
||||||
|
* <p>只删 {@code log_date} 早于 cutoff 的行:这条线正好是查询侧可见窗口的边界,
|
||||||
|
* 即"早已看不见、只剩占位"的行,删掉不影响任何读取路径。
|
||||||
|
*/
|
||||||
|
@Delete("""
|
||||||
|
DELETE FROM device_log_file
|
||||||
|
WHERE log_date < #{cutoff}
|
||||||
|
LIMIT #{batchSize}
|
||||||
|
""")
|
||||||
|
int deleteOlderThanBatch(@Param("cutoff") LocalDate cutoff, @Param("batchSize") int batchSize);
|
||||||
|
}
|
||||||
+32
@@ -0,0 +1,32 @@
|
|||||||
|
package com.nanri.aiimage.modules.devicelog.model.entity;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.annotation.IdType;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableId;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableName;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 日志采集配置:scope=global 为全局默认(source/device_id 存空串占位);
|
||||||
|
* scope=device 为终端级覆盖(按来源+设备精确命中,优先于全局)。
|
||||||
|
*/
|
||||||
|
@Data
|
||||||
|
@TableName("device_log_config")
|
||||||
|
public class DeviceLogConfigEntity {
|
||||||
|
|
||||||
|
@TableId(type = IdType.AUTO)
|
||||||
|
private Long id;
|
||||||
|
/** global / device。 */
|
||||||
|
private String scope;
|
||||||
|
/** 对应 device_log_file.source(global 行存空串)。 */
|
||||||
|
private String source;
|
||||||
|
/** 对应 device_log_file.device_id(global 行存空串)。 */
|
||||||
|
private String deviceId;
|
||||||
|
/** 覆盖行记录的设备展示名(列表展示用)。 */
|
||||||
|
private String deviceName;
|
||||||
|
/** full(全量)/ selected(精选)。 */
|
||||||
|
private String mode;
|
||||||
|
private LocalDateTime createdAt;
|
||||||
|
private LocalDateTime updatedAt;
|
||||||
|
}
|
||||||
+37
@@ -0,0 +1,37 @@
|
|||||||
|
package com.nanri.aiimage.modules.devicelog.model.entity;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.annotation.IdType;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableId;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableName;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.time.LocalDate;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 设备日志文件元数据(对象内容存独立 MinIO,本表只存索引与进度)。
|
||||||
|
* 一个「来源 + 设备 + 文件名 + 日志日期」一行,uploadedBytes/partCount 随增量上报推进。
|
||||||
|
*/
|
||||||
|
@Data
|
||||||
|
@TableName("device_log_file")
|
||||||
|
public class DeviceLogFileEntity {
|
||||||
|
|
||||||
|
@TableId(type = IdType.AUTO)
|
||||||
|
private Long id;
|
||||||
|
/** 来源:client(桌面客户端)/ maixiang(麦象采集机)。 */
|
||||||
|
private String source;
|
||||||
|
private String deviceId;
|
||||||
|
/** 展示名:客户端登录用户名或机器名。 */
|
||||||
|
private String deviceName;
|
||||||
|
/** 桌面客户端当前登录用户 id(users.id),maixiang 上报为空。 */
|
||||||
|
private Long uid;
|
||||||
|
/** 日志文件名(客户端可能含子目录,如 API/2026_09_15.log)。 */
|
||||||
|
private String fileName;
|
||||||
|
private LocalDate logDate;
|
||||||
|
/** 已上传的明文字节数(客户端增量断点由此对齐)。 */
|
||||||
|
private Long uploadedBytes;
|
||||||
|
private Integer partCount;
|
||||||
|
private LocalDateTime lastUploadAt;
|
||||||
|
private LocalDateTime createdAt;
|
||||||
|
private LocalDateTime updatedAt;
|
||||||
|
}
|
||||||
+19
@@ -0,0 +1,19 @@
|
|||||||
|
package com.nanri.aiimage.modules.devicelog.model.vo;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
/** 日志内容(按尾部截取)。 */
|
||||||
|
@Data
|
||||||
|
public class DeviceLogContentVo {
|
||||||
|
|
||||||
|
private Long fileId;
|
||||||
|
private String fileName;
|
||||||
|
/** 已解压的日志文本(自最早行边界起,保证不截出半行)。 */
|
||||||
|
private String content;
|
||||||
|
/** 服务端已收到的日志总字节数。 */
|
||||||
|
private long totalBytes;
|
||||||
|
/** 本次实际返回的字节数。 */
|
||||||
|
private long shownBytes;
|
||||||
|
/** true=内容被截断(更早的历史未返回,可加大 maxBytes 再取)。 */
|
||||||
|
private boolean truncated;
|
||||||
|
}
|
||||||
+25
@@ -0,0 +1,25 @@
|
|||||||
|
package com.nanri.aiimage.modules.devicelog.model.vo;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.time.LocalDate;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
/** 日志文件列表行。 */
|
||||||
|
@Data
|
||||||
|
public class DeviceLogFileVo {
|
||||||
|
|
||||||
|
private Long id;
|
||||||
|
private String source;
|
||||||
|
private String deviceId;
|
||||||
|
private String deviceName;
|
||||||
|
/** 关联用户展示名(users.username;解析不到时为空)。 */
|
||||||
|
private String username;
|
||||||
|
private Long uid;
|
||||||
|
private String fileName;
|
||||||
|
private LocalDate logDate;
|
||||||
|
private Long uploadedBytes;
|
||||||
|
private Integer partCount;
|
||||||
|
private LocalDateTime lastUploadAt;
|
||||||
|
private LocalDateTime createdAt;
|
||||||
|
}
|
||||||
+17
@@ -0,0 +1,17 @@
|
|||||||
|
package com.nanri.aiimage.modules.devicelog.model.vo;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/** 日志文件分页结果。 */
|
||||||
|
@Data
|
||||||
|
public class DeviceLogPageVo {
|
||||||
|
|
||||||
|
private List<DeviceLogFileVo> items;
|
||||||
|
private long total;
|
||||||
|
private long page;
|
||||||
|
private long pageSize;
|
||||||
|
/** 服务端保留天数(前端提示「日志仅保留 N 天」)。 */
|
||||||
|
private int retentionDays;
|
||||||
|
}
|
||||||
+169
@@ -0,0 +1,169 @@
|
|||||||
|
package com.nanri.aiimage.modules.devicelog.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.modules.devicelog.mapper.DeviceLogConfigMapper;
|
||||||
|
import com.nanri.aiimage.modules.devicelog.model.entity.DeviceLogConfigEntity;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 日志采集配置:全局默认 + 终端覆盖(超管在后台「日志管理」调整)。
|
||||||
|
*
|
||||||
|
* <p>上报端(桌面客户端 / 麦象)定期拉取生效配置:全量上传目录内全部日志;
|
||||||
|
* 精选模式只上传关键日志(排除清单见 {@link #selectedExcludes},随配置一起下发,
|
||||||
|
* 调整清单无需发客户端版本)。
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class DeviceLogConfigService {
|
||||||
|
|
||||||
|
public static final String MODE_FULL = "full";
|
||||||
|
public static final String MODE_SELECTED = "selected";
|
||||||
|
|
||||||
|
/** 精选模式排除清单(glob,按来源;相对日志目录的文件名)。 */
|
||||||
|
private static final Map<String, List<String>> SELECTED_EXCLUDES = Map.of(
|
||||||
|
"client", List.of("pywebview.log"),
|
||||||
|
"maixiang", List.of("kk-browser.log*", "*_console.log", "test*.log"));
|
||||||
|
|
||||||
|
private final DeviceLogConfigMapper deviceLogConfigMapper;
|
||||||
|
|
||||||
|
/** 生效配置(终端覆盖 > 全局默认 > 兜底全量)。 */
|
||||||
|
public record EffectiveConfig(String mode, List<String> exclude) {
|
||||||
|
}
|
||||||
|
|
||||||
|
public EffectiveConfig resolve(String source, String deviceId) {
|
||||||
|
DeviceLogConfigEntity override = deviceLogConfigMapper.selectOne(
|
||||||
|
new LambdaQueryWrapper<DeviceLogConfigEntity>()
|
||||||
|
.eq(DeviceLogConfigEntity::getScope, "device")
|
||||||
|
.eq(DeviceLogConfigEntity::getSource, source)
|
||||||
|
.eq(DeviceLogConfigEntity::getDeviceId, deviceId)
|
||||||
|
.last("limit 1"));
|
||||||
|
String mode = override != null ? override.getMode() : globalMode();
|
||||||
|
return new EffectiveConfig(mode, selectedExcludes(mode, source));
|
||||||
|
}
|
||||||
|
|
||||||
|
public String globalMode() {
|
||||||
|
DeviceLogConfigEntity global = findGlobal();
|
||||||
|
return global == null ? MODE_FULL : global.getMode();
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<String> selectedExcludes(String mode, String source) {
|
||||||
|
if (!MODE_SELECTED.equals(mode)) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
List<String> excludes = SELECTED_EXCLUDES.get(source);
|
||||||
|
if (excludes == null) {
|
||||||
|
log.warn("[device-log] 来源 {} 无精选排除清单,精选模式将等价全量", source);
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
return new ArrayList<>(excludes);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setGlobalMode(String mode) {
|
||||||
|
String safeMode = normalizeMode(mode);
|
||||||
|
DeviceLogConfigEntity global = findGlobal();
|
||||||
|
if (global == null) {
|
||||||
|
global = new DeviceLogConfigEntity();
|
||||||
|
global.setScope("global");
|
||||||
|
global.setSource("");
|
||||||
|
global.setDeviceId("");
|
||||||
|
global.setMode(safeMode);
|
||||||
|
deviceLogConfigMapper.insert(global);
|
||||||
|
log.info("[device-log] 全局采集模式初始化 mode={}", safeMode);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (safeMode.equals(global.getMode())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
deviceLogConfigMapper.updateById(withMode(global, safeMode));
|
||||||
|
log.info("[device-log] 全局采集模式更新 {} → {}", global.getMode(), safeMode);
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<DeviceLogConfigEntity> listOverrides(String keyword) {
|
||||||
|
LambdaQueryWrapper<DeviceLogConfigEntity> qw = new LambdaQueryWrapper<DeviceLogConfigEntity>()
|
||||||
|
.eq(DeviceLogConfigEntity::getScope, "device");
|
||||||
|
if (keyword != null && !keyword.isBlank()) {
|
||||||
|
String kw = keyword.trim();
|
||||||
|
qw.and(w -> w.like(DeviceLogConfigEntity::getDeviceId, kw)
|
||||||
|
.or().like(DeviceLogConfigEntity::getDeviceName, kw));
|
||||||
|
}
|
||||||
|
return deviceLogConfigMapper.selectList(qw
|
||||||
|
.orderByDesc(DeviceLogConfigEntity::getUpdatedAt)
|
||||||
|
.last("limit 500"));
|
||||||
|
}
|
||||||
|
|
||||||
|
public DeviceLogConfigEntity upsertOverride(String source, String deviceId, String deviceName, String mode) {
|
||||||
|
String safeSource = requireText(source, "source", 32);
|
||||||
|
String safeDeviceId = requireText(deviceId, "deviceId", 128);
|
||||||
|
String safeMode = normalizeMode(mode);
|
||||||
|
DeviceLogConfigEntity row = deviceLogConfigMapper.selectOne(
|
||||||
|
new LambdaQueryWrapper<DeviceLogConfigEntity>()
|
||||||
|
.eq(DeviceLogConfigEntity::getScope, "device")
|
||||||
|
.eq(DeviceLogConfigEntity::getSource, safeSource)
|
||||||
|
.eq(DeviceLogConfigEntity::getDeviceId, safeDeviceId)
|
||||||
|
.last("limit 1"));
|
||||||
|
if (row == null) {
|
||||||
|
row = new DeviceLogConfigEntity();
|
||||||
|
row.setScope("device");
|
||||||
|
row.setSource(safeSource);
|
||||||
|
row.setDeviceId(safeDeviceId);
|
||||||
|
row.setDeviceName(deviceName);
|
||||||
|
row.setMode(safeMode);
|
||||||
|
deviceLogConfigMapper.insert(row);
|
||||||
|
log.info("[device-log] 新增终端覆盖 source={} device={} mode={}", safeSource, safeDeviceId, safeMode);
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
DeviceLogConfigEntity update = withMode(row, safeMode);
|
||||||
|
if (deviceName != null && !deviceName.isBlank()) {
|
||||||
|
update.setDeviceName(deviceName);
|
||||||
|
}
|
||||||
|
deviceLogConfigMapper.updateById(update);
|
||||||
|
log.info("[device-log] 终端覆盖更新 source={} device={} mode={}", safeSource, safeDeviceId, safeMode);
|
||||||
|
return update;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean deleteOverride(Long id) {
|
||||||
|
if (id == null) {
|
||||||
|
throw new BusinessException(400, "id 不能为空");
|
||||||
|
}
|
||||||
|
int deleted = deviceLogConfigMapper.deleteById(id);
|
||||||
|
log.info("[device-log] 终端覆盖删除 id={} deleted={}", id, deleted);
|
||||||
|
return deleted > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DeviceLogConfigEntity findGlobal() {
|
||||||
|
return deviceLogConfigMapper.selectOne(new LambdaQueryWrapper<DeviceLogConfigEntity>()
|
||||||
|
.eq(DeviceLogConfigEntity::getScope, "global")
|
||||||
|
.last("limit 1"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static DeviceLogConfigEntity withMode(DeviceLogConfigEntity row, String mode) {
|
||||||
|
DeviceLogConfigEntity update = new DeviceLogConfigEntity();
|
||||||
|
update.setId(row.getId());
|
||||||
|
update.setMode(mode);
|
||||||
|
return update;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String normalizeMode(String mode) {
|
||||||
|
String trimmed = mode == null ? "" : mode.trim().toLowerCase();
|
||||||
|
if (!MODE_FULL.equals(trimmed) && !MODE_SELECTED.equals(trimmed)) {
|
||||||
|
throw new BusinessException(400, "mode 只支持 full / selected");
|
||||||
|
}
|
||||||
|
return trimmed;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String requireText(String value, String field, int maxLength) {
|
||||||
|
String trimmed = value == null ? "" : value.trim();
|
||||||
|
if (trimmed.isEmpty() || trimmed.length() > maxLength) {
|
||||||
|
throw new BusinessException(400, field + " 非法");
|
||||||
|
}
|
||||||
|
return trimmed;
|
||||||
|
}
|
||||||
|
}
|
||||||
+77
@@ -0,0 +1,77 @@
|
|||||||
|
package com.nanri.aiimage.modules.devicelog.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
|
import com.nanri.aiimage.config.DeviceLogOssProperties;
|
||||||
|
import com.nanri.aiimage.modules.devicelog.mapper.DeviceLogFileMapper;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.scheduling.annotation.Scheduled;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.LocalDate;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 设备日志元数据的保留期清理。
|
||||||
|
*
|
||||||
|
* <p>对象侧一直有生命周期(主机B 独立 MinIO 桶,由运维 mc 定时任务按 7 天回收),
|
||||||
|
* 但 {@code device_log_file} 的元数据行此前**只增不删**:客户端每 60 秒上报一轮,
|
||||||
|
* 每台设备每天登记若干「来源+设备+文件名+日期」行,长期运行会无限累积。
|
||||||
|
*
|
||||||
|
* <p>查询侧本就只展示保留期内的行({@link DeviceLogService#page} 用同一个
|
||||||
|
* {@code retentionDays} 过滤),这里按完全相同的边界删掉早已不可见的行。
|
||||||
|
* 保留天数直接复用 {@code aiimage.device-log-oss.retention-days},
|
||||||
|
* 与对象侧共用同一个配置项,避免两边各写一份天数后悄悄漂移。
|
||||||
|
*
|
||||||
|
* <p>对象本身仍由桶生命周期负责回收,本任务不碰对象:两条时间线按「对象修改时间」
|
||||||
|
* 与「日志日期」衡量,可能有几天错位,但对象最终仍会被桶规则删除,不会永久残留。
|
||||||
|
*
|
||||||
|
* <p>双节点用 job 锁保证单实例执行;分批删除并限制单轮批次数,避免一次跑太久占住锁。
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class DeviceLogRetentionService {
|
||||||
|
|
||||||
|
/** 单轮最多删除的批次数(每批 batchSize 行),剩余留给下一轮。 */
|
||||||
|
static final int MAX_BATCHES_PER_RUN = 20;
|
||||||
|
|
||||||
|
private final DeviceLogFileMapper deviceLogFileMapper;
|
||||||
|
private final DeviceLogOssProperties properties;
|
||||||
|
private final DistributedJobLockService distributedJobLockService;
|
||||||
|
|
||||||
|
@Value("${aiimage.device-log.retention-batch-size:2000}")
|
||||||
|
private int retentionBatchSize = 2000;
|
||||||
|
|
||||||
|
@Scheduled(cron = "${aiimage.device-log.retention-cron:0 40 4 * * *}")
|
||||||
|
public void purgeExpiredMetadata() {
|
||||||
|
int days = properties.retentionDaysOrDefault();
|
||||||
|
int batchSize = Math.max(100, retentionBatchSize);
|
||||||
|
// 与查询侧可见窗口同一条边界:page() 只展示 log_date >= 今天-(days-1) 的行
|
||||||
|
LocalDate cutoff = LocalDate.now().minusDays(days - 1L);
|
||||||
|
|
||||||
|
DistributedJobLockService.LockHandle lockHandle =
|
||||||
|
distributedJobLockService.tryLock("device-log:metadata-retention", Duration.ofMinutes(15));
|
||||||
|
if (lockHandle == null) {
|
||||||
|
log.info("[device-log] 元数据保留清理跳过:另一实例持锁");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try (lockHandle) {
|
||||||
|
int totalDeleted = 0;
|
||||||
|
int batches = 0;
|
||||||
|
while (batches < MAX_BATCHES_PER_RUN) {
|
||||||
|
int deleted = deviceLogFileMapper.deleteOlderThanBatch(cutoff, batchSize);
|
||||||
|
batches++;
|
||||||
|
totalDeleted += deleted;
|
||||||
|
if (deleted < batchSize) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.info("[device-log] 元数据保留清理完成 cutoff={} retentionDays={} deleted={} batches={}",
|
||||||
|
cutoff, days, totalDeleted, batches);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[device-log] 元数据保留清理失败 cutoff={} msg={}", cutoff, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+458
@@ -0,0 +1,458 @@
|
|||||||
|
package com.nanri.aiimage.modules.devicelog.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.common.mapper.AdminUserMapper;
|
||||||
|
import com.nanri.aiimage.common.model.entity.AdminUserEntity;
|
||||||
|
import com.nanri.aiimage.config.DeviceLogOssProperties;
|
||||||
|
import com.nanri.aiimage.modules.devicelog.mapper.DeviceLogFileMapper;
|
||||||
|
import com.nanri.aiimage.modules.devicelog.model.entity.DeviceLogFileEntity;
|
||||||
|
import com.nanri.aiimage.modules.devicelog.model.vo.DeviceLogContentVo;
|
||||||
|
import com.nanri.aiimage.modules.devicelog.model.vo.DeviceLogFileVo;
|
||||||
|
import com.nanri.aiimage.modules.devicelog.model.vo.DeviceLogPageVo;
|
||||||
|
import com.nanri.aiimage.modules.devicelog.storage.DeviceLogStorageService;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.io.ByteArrayInputStream;
|
||||||
|
import java.io.ByteArrayOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.time.LocalDate;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.ArrayDeque;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Deque;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.function.Function;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
import java.util.zip.GZIPInputStream;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 设备日志:增量片段接收(桌面客户端 / 麦象采集机上报)与后台查询。
|
||||||
|
*
|
||||||
|
* <p>存储模型:一个「来源+设备+文件名+日期」一行元数据;内容以 gzip 片段对象按
|
||||||
|
* 「起始偏移」命名存独立 MinIO(device-logs/…/{offset}.log.gz)。客户端按本地
|
||||||
|
* uploadedBytes 断点续传,服务端条件推进偏移;查看/下载时按偏移顺序拼接解压。
|
||||||
|
* 片段 key 带偏移(而非序号),重复上报与乱序重试都会覆盖同一对象,天然幂等。
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class DeviceLogService {
|
||||||
|
|
||||||
|
/** 单片段明文上限(客户端按 2MB 切片,此处留防线余量)。 */
|
||||||
|
private static final long MAX_PART_PLAIN_BYTES = 8L * 1024 * 1024;
|
||||||
|
private static final long MAX_PART_GZIP_BYTES = 8L * 1024 * 1024;
|
||||||
|
|
||||||
|
private static final long DEFAULT_TAIL_BYTES = 256L * 1024;
|
||||||
|
private static final long MIN_TAIL_BYTES = 16L * 1024;
|
||||||
|
private static final long MAX_TAIL_BYTES = 8L * 1024 * 1024;
|
||||||
|
|
||||||
|
private static final long MAX_PAGE_SIZE = 100L;
|
||||||
|
|
||||||
|
/** 来源白名单形态:小写字母开头,长度 ≤32(client / maixiang / 未来新来源)。 */
|
||||||
|
private static final Pattern SOURCE_PATTERN = Pattern.compile("^[a-z][a-z0-9_-]{0,31}$");
|
||||||
|
/** 对象 key 段落清洗:路径分隔符、通配符、控制字符一律换成下划线。 */
|
||||||
|
private static final Pattern UNSAFE_SEGMENT = Pattern.compile("[\\\\/:*?\"<>|\\x00-\\x1F]+");
|
||||||
|
|
||||||
|
private final DeviceLogFileMapper deviceLogFileMapper;
|
||||||
|
private final AdminUserMapper adminUserMapper;
|
||||||
|
private final DeviceLogStorageService storage;
|
||||||
|
private final DeviceLogOssProperties properties;
|
||||||
|
|
||||||
|
/** 上报处理结果。 */
|
||||||
|
public record PartResult(long uploadedBytes, int partCount, boolean accepted, boolean skipped) {
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------- 上报
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 接收一个增量片段。
|
||||||
|
*
|
||||||
|
* @param offset 客户端认为的已上传明文偏移(必须等于服务端记录值才能追加)
|
||||||
|
* @param plainBytes 本片段解压后的明文字节数(客户端告知;服务端只存 gzip 不解压)
|
||||||
|
*/
|
||||||
|
public PartResult recordPart(String source, String deviceId, String deviceName, Long uid,
|
||||||
|
String fileName, LocalDate logDate, long offset, long plainBytes,
|
||||||
|
byte[] gzipBytes) {
|
||||||
|
String safeSource = normalizeSource(source);
|
||||||
|
String safeDeviceId = requireSegment(deviceId, "deviceId", 128);
|
||||||
|
String safeFileName = requireSegment(fileName, "fileName", 255);
|
||||||
|
if (logDate == null) {
|
||||||
|
throw new BusinessException(400, "logDate 不能为空");
|
||||||
|
}
|
||||||
|
if (offset < 0) {
|
||||||
|
throw new BusinessException(400, "offset 非法");
|
||||||
|
}
|
||||||
|
if (plainBytes <= 0 || plainBytes > MAX_PART_PLAIN_BYTES) {
|
||||||
|
throw new BusinessException(400, "plainBytes 非法(1 ~ " + MAX_PART_PLAIN_BYTES + ")");
|
||||||
|
}
|
||||||
|
if (gzipBytes == null || gzipBytes.length == 0 || gzipBytes.length > MAX_PART_GZIP_BYTES) {
|
||||||
|
throw new BusinessException(400, "片段内容为空或超过上限");
|
||||||
|
}
|
||||||
|
if (!storage.enabled()) {
|
||||||
|
log.error("[device-log] 上报被拒:对象存储未就绪 source={} device={} file={}", safeSource, safeDeviceId, safeFileName);
|
||||||
|
throw new BusinessException(503, "日志存储未就绪,请稍后重试");
|
||||||
|
}
|
||||||
|
|
||||||
|
DeviceLogFileEntity row = findOrCreate(safeSource, safeDeviceId, safeFileName, logDate, deviceName, uid);
|
||||||
|
long current = row.getUploadedBytes() == null ? 0L : row.getUploadedBytes();
|
||||||
|
|
||||||
|
if (offset < current) {
|
||||||
|
// 重试/重复上报:内容已收过,幂等跳过(返回服务端权威进度供客户端对齐)
|
||||||
|
log.info("[device-log] 片段重复,幂等跳过 source={} device={} file={} date={} offset={} current={}",
|
||||||
|
safeSource, safeDeviceId, safeFileName, logDate, offset, current);
|
||||||
|
return new PartResult(current, nvl(row.getPartCount()), false, true);
|
||||||
|
}
|
||||||
|
if (offset > current) {
|
||||||
|
// 出现空洞(客户端本地进度领先于服务端):拒绝,让客户端从服务端进度重读
|
||||||
|
log.warn("[device-log] 片段偏移不连续 source={} device={} file={} date={} offset={} current={}",
|
||||||
|
safeSource, safeDeviceId, safeFileName, logDate, offset, current);
|
||||||
|
throw new BusinessException(409, "偏移不连续,请从 uploadedBytes=" + current + " 重新读取");
|
||||||
|
}
|
||||||
|
|
||||||
|
String objectKey = objectKeyPrefix(safeSource, safeDeviceId, logDate, safeFileName)
|
||||||
|
+ String.format("%012d.log.gz", offset);
|
||||||
|
storage.putPart(objectKey, gzipBytes);
|
||||||
|
|
||||||
|
// 条件推进(uploaded_bytes 与读取时一致才更新;双节点并发时另一方以 0 行影响放弃,以库中值为准)
|
||||||
|
int updated = deviceLogFileMapper.update(null, new LambdaUpdateWrapper<DeviceLogFileEntity>()
|
||||||
|
.eq(DeviceLogFileEntity::getId, row.getId())
|
||||||
|
.eq(DeviceLogFileEntity::getUploadedBytes, current)
|
||||||
|
.set(DeviceLogFileEntity::getUploadedBytes, offset + plainBytes)
|
||||||
|
.setSql("part_count = part_count + 1")
|
||||||
|
.set(DeviceLogFileEntity::getLastUploadAt, LocalDateTime.now())
|
||||||
|
.set(deviceName != null && !deviceName.isBlank(), DeviceLogFileEntity::getDeviceName, deviceName)
|
||||||
|
.set(uid != null, DeviceLogFileEntity::getUid, uid));
|
||||||
|
if (updated <= 0) {
|
||||||
|
DeviceLogFileEntity latest = deviceLogFileMapper.selectById(row.getId());
|
||||||
|
long latestBytes = latest == null || latest.getUploadedBytes() == null ? current : latest.getUploadedBytes();
|
||||||
|
log.warn("[device-log] 并发推进冲突,以库中值为准 id={} offset={} 库中={}", row.getId(), offset, latestBytes);
|
||||||
|
return new PartResult(latestBytes, latest == null ? 0 : nvl(latest.getPartCount()), false, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
long after = offset + plainBytes;
|
||||||
|
log.info("[device-log] 已收片段 source={} device={} file={} date={} offset={} +{}B → {}B key={}",
|
||||||
|
safeSource, safeDeviceId, safeFileName, logDate, offset, plainBytes, after, objectKey);
|
||||||
|
return new PartResult(after, nvl(row.getPartCount()) + 1, true, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 客户端进度对齐:返回服务端已持有的偏移与片段数。 */
|
||||||
|
public Map<String, Object> state(String source, String deviceId, String fileName, LocalDate logDate) {
|
||||||
|
String safeSource = normalizeSource(source);
|
||||||
|
DeviceLogFileEntity row = find(safeSource, requireSegment(deviceId, "deviceId", 128),
|
||||||
|
requireSegment(fileName, "fileName", 255), logDate);
|
||||||
|
return Map.of(
|
||||||
|
"exists", row != null,
|
||||||
|
"uploadedBytes", row == null || row.getUploadedBytes() == null ? 0L : row.getUploadedBytes(),
|
||||||
|
"parts", row == null ? 0 : nvl(row.getPartCount()));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------- 查询
|
||||||
|
|
||||||
|
public DeviceLogPageVo page(String source, String keyword, LocalDate startDate, LocalDate endDate,
|
||||||
|
Long pageParam, Long pageSizeParam) {
|
||||||
|
int retentionDays = properties.retentionDaysOrDefault();
|
||||||
|
LocalDate minDate = LocalDate.now().minusDays(retentionDays - 1L);
|
||||||
|
LocalDate from = startDate == null || startDate.isBefore(minDate) ? minDate : startDate;
|
||||||
|
long safePage = pageParam == null || pageParam < 1 ? 1L : pageParam;
|
||||||
|
long safeSize = pageSizeParam == null || pageSizeParam < 1
|
||||||
|
? 20L : Math.min(pageSizeParam, MAX_PAGE_SIZE);
|
||||||
|
|
||||||
|
Function<Boolean, LambdaQueryWrapper<DeviceLogFileEntity>> wrapperBuilder = countOnly -> {
|
||||||
|
LambdaQueryWrapper<DeviceLogFileEntity> qw = new LambdaQueryWrapper<>();
|
||||||
|
if (source != null && !source.isBlank()) {
|
||||||
|
qw.eq(DeviceLogFileEntity::getSource, source.trim());
|
||||||
|
}
|
||||||
|
if (keyword != null && !keyword.isBlank()) {
|
||||||
|
String kw = keyword.trim();
|
||||||
|
qw.and(w -> w.like(DeviceLogFileEntity::getDeviceName, kw)
|
||||||
|
.or().like(DeviceLogFileEntity::getDeviceId, kw)
|
||||||
|
.or().like(DeviceLogFileEntity::getFileName, kw));
|
||||||
|
}
|
||||||
|
qw.ge(DeviceLogFileEntity::getLogDate, from);
|
||||||
|
if (endDate != null) {
|
||||||
|
qw.le(DeviceLogFileEntity::getLogDate, endDate);
|
||||||
|
}
|
||||||
|
return qw;
|
||||||
|
};
|
||||||
|
|
||||||
|
Long totalValue = deviceLogFileMapper.selectCount(wrapperBuilder.apply(true));
|
||||||
|
long total = totalValue == null ? 0L : totalValue;
|
||||||
|
long offset = Math.max(0L, (safePage - 1) * safeSize);
|
||||||
|
List<DeviceLogFileEntity> rows = total == 0 ? List.of()
|
||||||
|
: deviceLogFileMapper.selectList(wrapperBuilder.apply(false)
|
||||||
|
.orderByDesc(DeviceLogFileEntity::getLastUploadAt)
|
||||||
|
.orderByDesc(DeviceLogFileEntity::getId)
|
||||||
|
.last("limit " + offset + "," + safeSize));
|
||||||
|
|
||||||
|
Map<Long, String> usernameOf = resolveUsernames(rows);
|
||||||
|
List<DeviceLogFileVo> items = new ArrayList<>(rows.size());
|
||||||
|
for (DeviceLogFileEntity row : rows) {
|
||||||
|
items.add(toVo(row, usernameOf));
|
||||||
|
}
|
||||||
|
|
||||||
|
DeviceLogPageVo vo = new DeviceLogPageVo();
|
||||||
|
vo.setItems(items);
|
||||||
|
vo.setTotal(total);
|
||||||
|
vo.setPage(safePage);
|
||||||
|
vo.setPageSize(safeSize);
|
||||||
|
vo.setRetentionDays(retentionDays);
|
||||||
|
log.info("[device-log] 列表查询 source={} keyword={} 起={} 止={} page={} size={} 命中={}",
|
||||||
|
source, keyword, from, endDate, safePage, safeSize, total);
|
||||||
|
return vo;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 最近上报过的终端(来源+设备去重,供后台配置终端覆盖时选择)。 */
|
||||||
|
public List<Map<String, Object>> recentDevices() {
|
||||||
|
LocalDate minDate = LocalDate.now().minusDays(properties.retentionDaysOrDefault() - 1L);
|
||||||
|
return deviceLogFileMapper.selectMaps(new QueryWrapper<DeviceLogFileEntity>()
|
||||||
|
.select("source",
|
||||||
|
"device_id AS deviceId",
|
||||||
|
"MAX(device_name) AS deviceName",
|
||||||
|
"MAX(last_upload_at) AS lastUploadAt")
|
||||||
|
.ge("log_date", minDate)
|
||||||
|
.groupBy("source", "device_id")
|
||||||
|
.orderByDesc("lastUploadAt")
|
||||||
|
.last("limit 200"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 尾部内容:从最新片段往前读,尽量凑满 maxBytes(自最早行边界起截取,不出现半行)。 */
|
||||||
|
public DeviceLogContentVo readTail(Long fileId, Long maxBytesParam) {
|
||||||
|
DeviceLogFileEntity row = requireFile(fileId);
|
||||||
|
long maxBytes = maxBytesParam == null ? DEFAULT_TAIL_BYTES
|
||||||
|
: Math.max(MIN_TAIL_BYTES, Math.min(maxBytesParam, MAX_TAIL_BYTES));
|
||||||
|
String prefix = objectKeyPrefix(row);
|
||||||
|
List<String> partKeys = storage.listParts(prefix);
|
||||||
|
|
||||||
|
Deque<byte[]> chunks = new ArrayDeque<>();
|
||||||
|
long acc = 0;
|
||||||
|
int idx = partKeys.size() - 1;
|
||||||
|
for (; idx >= 0 && acc < maxBytes; idx--) {
|
||||||
|
byte[] plain;
|
||||||
|
try {
|
||||||
|
plain = gunzip(storage.readPartBytes(partKeys.get(idx)));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[device-log] 片段读取/解压失败,跳过 key={} err={}", partKeys.get(idx), ex.getMessage());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
chunks.addFirst(plain);
|
||||||
|
acc += plain.length;
|
||||||
|
}
|
||||||
|
boolean truncated = idx >= 0;
|
||||||
|
|
||||||
|
ByteArrayOutputStream merged = new ByteArrayOutputStream((int) Math.min(acc, Integer.MAX_VALUE));
|
||||||
|
for (byte[] chunk : chunks) {
|
||||||
|
merged.write(chunk, 0, chunk.length);
|
||||||
|
}
|
||||||
|
byte[] bytes = merged.toByteArray();
|
||||||
|
if (bytes.length > maxBytes) {
|
||||||
|
int cut = (int) (bytes.length - maxBytes);
|
||||||
|
int nl = indexOfNewline(bytes, cut);
|
||||||
|
// 从行边界开始截(不留半行);但若这样会切掉全部内容(超长行),退回按字节截
|
||||||
|
if (nl >= 0 && nl + 1 < bytes.length) {
|
||||||
|
cut = nl + 1;
|
||||||
|
}
|
||||||
|
bytes = Arrays.copyOfRange(bytes, cut, bytes.length);
|
||||||
|
truncated = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
DeviceLogContentVo vo = new DeviceLogContentVo();
|
||||||
|
vo.setFileId(row.getId());
|
||||||
|
vo.setFileName(row.getFileName());
|
||||||
|
vo.setContent(new String(bytes, StandardCharsets.UTF_8));
|
||||||
|
vo.setTotalBytes(row.getUploadedBytes() == null ? 0L : row.getUploadedBytes());
|
||||||
|
vo.setShownBytes(bytes.length);
|
||||||
|
vo.setTruncated(truncated);
|
||||||
|
log.info("[device-log] 内容查看 id={} file={} 总大小={}B 返回={}B 截断={}",
|
||||||
|
row.getId(), row.getFileName(), vo.getTotalBytes(), bytes.length, truncated);
|
||||||
|
return vo;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 按偏移顺序流式拼接全部片段(解压后写响应,无整文件内存占用)。 */
|
||||||
|
public void streamDownload(Long fileId, OutputStream out) throws IOException {
|
||||||
|
DeviceLogFileEntity row = requireFile(fileId);
|
||||||
|
List<String> partKeys = storage.listParts(objectKeyPrefix(row));
|
||||||
|
if (partKeys.isEmpty()) {
|
||||||
|
throw new BusinessException(404, "该日志暂无内容");
|
||||||
|
}
|
||||||
|
for (String key : partKeys) {
|
||||||
|
try (InputStream raw = storage.openPartStream(key);
|
||||||
|
GZIPInputStream gz = new GZIPInputStream(raw)) {
|
||||||
|
gz.transferTo(out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.flush();
|
||||||
|
log.info("[device-log] 下载拼接完成 id={} file={} 片段数={}", row.getId(), row.getFileName(), partKeys.size());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 删除日志文件(片段对象 + 元数据行)。返回删除的对象数。 */
|
||||||
|
public int deleteFile(Long fileId) {
|
||||||
|
DeviceLogFileEntity row = requireFile(fileId);
|
||||||
|
List<String> partKeys = storage.listParts(objectKeyPrefix(row));
|
||||||
|
List<String> failed = storage.deleteParts(partKeys);
|
||||||
|
deviceLogFileMapper.deleteById(fileId);
|
||||||
|
log.info("[device-log] 删除日志 id={} file={} 片段总数={} 删除失败={}",
|
||||||
|
row.getId(), row.getFileName(), partKeys.size(), failed.size());
|
||||||
|
return partKeys.size() - failed.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------- 内部
|
||||||
|
|
||||||
|
/** 按 id 取日志文件行(不存在抛 404)。 */
|
||||||
|
public DeviceLogFileEntity requireFile(Long fileId) {
|
||||||
|
if (fileId == null) {
|
||||||
|
throw new BusinessException(400, "fileId 不能为空");
|
||||||
|
}
|
||||||
|
DeviceLogFileEntity row = deviceLogFileMapper.selectById(fileId);
|
||||||
|
if (row == null) {
|
||||||
|
throw new BusinessException(404, "日志文件不存在或已清理");
|
||||||
|
}
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DeviceLogFileEntity findOrCreate(String source, String deviceId, String fileName,
|
||||||
|
LocalDate logDate, String deviceName, Long uid) {
|
||||||
|
DeviceLogFileEntity row = find(source, deviceId, fileName, logDate);
|
||||||
|
if (row != null) {
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
DeviceLogFileEntity entity = new DeviceLogFileEntity();
|
||||||
|
entity.setSource(source);
|
||||||
|
entity.setDeviceId(deviceId);
|
||||||
|
entity.setFileName(fileName);
|
||||||
|
entity.setLogDate(logDate);
|
||||||
|
entity.setDeviceName(deviceName);
|
||||||
|
entity.setUid(uid);
|
||||||
|
entity.setUploadedBytes(0L);
|
||||||
|
entity.setPartCount(0);
|
||||||
|
try {
|
||||||
|
deviceLogFileMapper.insert(entity);
|
||||||
|
log.info("[device-log] 登记新日志文件 id={} source={} device={} file={} date={}",
|
||||||
|
entity.getId(), source, deviceId, fileName, logDate);
|
||||||
|
return entity;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
// 双节点并发首传同一文件:唯一键冲突后复用已有行
|
||||||
|
DeviceLogFileEntity existing = find(source, deviceId, fileName, logDate);
|
||||||
|
if (existing != null) {
|
||||||
|
log.info("[device-log] 并发登记同一文件,复用已有行 id={} file={}", existing.getId(), fileName);
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
throw ex;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private DeviceLogFileEntity find(String source, String deviceId, String fileName, LocalDate logDate) {
|
||||||
|
if (logDate == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return deviceLogFileMapper.selectOne(new LambdaQueryWrapper<DeviceLogFileEntity>()
|
||||||
|
.eq(DeviceLogFileEntity::getSource, source)
|
||||||
|
.eq(DeviceLogFileEntity::getDeviceId, deviceId)
|
||||||
|
.eq(DeviceLogFileEntity::getFileName, fileName)
|
||||||
|
.eq(DeviceLogFileEntity::getLogDate, logDate)
|
||||||
|
.last("limit 1"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private String objectKeyPrefix(DeviceLogFileEntity row) {
|
||||||
|
return objectKeyPrefix(row.getSource(), row.getDeviceId(), row.getLogDate(), row.getFileName());
|
||||||
|
}
|
||||||
|
|
||||||
|
private String objectKeyPrefix(String source, String deviceId, LocalDate logDate, String fileName) {
|
||||||
|
return String.format("device-logs/%s/%s/%s/%s/",
|
||||||
|
safeSegment(source), safeSegment(deviceId), logDate, safeSegment(fileName));
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<Long, String> resolveUsernames(List<DeviceLogFileEntity> rows) {
|
||||||
|
List<Long> uids = rows.stream()
|
||||||
|
.map(DeviceLogFileEntity::getUid)
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.distinct()
|
||||||
|
.toList();
|
||||||
|
if (uids.isEmpty()) {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
List<AdminUserEntity> users = adminUserMapper.selectBatchIds(uids);
|
||||||
|
Map<Long, String> map = new java.util.HashMap<>();
|
||||||
|
for (AdminUserEntity user : users) {
|
||||||
|
map.put(user.getId(), user.getUsername());
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DeviceLogFileVo toVo(DeviceLogFileEntity row, Map<Long, String> usernameOf) {
|
||||||
|
DeviceLogFileVo vo = new DeviceLogFileVo();
|
||||||
|
vo.setId(row.getId());
|
||||||
|
vo.setSource(row.getSource());
|
||||||
|
vo.setDeviceId(row.getDeviceId());
|
||||||
|
vo.setDeviceName(row.getDeviceName());
|
||||||
|
vo.setUid(row.getUid());
|
||||||
|
vo.setUsername(row.getUid() == null ? null : usernameOf.get(row.getUid()));
|
||||||
|
vo.setFileName(row.getFileName());
|
||||||
|
vo.setLogDate(row.getLogDate());
|
||||||
|
vo.setUploadedBytes(row.getUploadedBytes());
|
||||||
|
vo.setPartCount(row.getPartCount());
|
||||||
|
vo.setLastUploadAt(row.getLastUploadAt());
|
||||||
|
vo.setCreatedAt(row.getCreatedAt());
|
||||||
|
return vo;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeSource(String source) {
|
||||||
|
String trimmed = source == null ? "" : source.trim();
|
||||||
|
if (!SOURCE_PATTERN.matcher(trimmed).matches()) {
|
||||||
|
log.warn("[device-log] 非法来源被拒 source={}", source);
|
||||||
|
throw new BusinessException(400, "source 非法(小写字母开头,数字/下划线/中划线,≤32)");
|
||||||
|
}
|
||||||
|
return trimmed;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String requireSegment(String value, String field, int maxLength) {
|
||||||
|
String trimmed = value == null ? "" : value.trim();
|
||||||
|
if (trimmed.isEmpty()) {
|
||||||
|
throw new BusinessException(400, field + " 不能为空");
|
||||||
|
}
|
||||||
|
if (trimmed.length() > maxLength) {
|
||||||
|
throw new BusinessException(400, field + " 超长(>" + maxLength + ")");
|
||||||
|
}
|
||||||
|
return trimmed;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 清洗对象 key 段落:路径分隔符等换成下划线,剔除「.」「..」防穿越。 */
|
||||||
|
private String safeSegment(String value) {
|
||||||
|
String cleaned = UNSAFE_SEGMENT.matcher(value == null ? "" : value.trim()).replaceAll("_");
|
||||||
|
if (cleaned.isEmpty() || cleaned.equals(".") || cleaned.equals("..")) {
|
||||||
|
return "_";
|
||||||
|
}
|
||||||
|
return cleaned;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static byte[] gunzip(byte[] gz) throws IOException {
|
||||||
|
try (GZIPInputStream in = new GZIPInputStream(new ByteArrayInputStream(gz));
|
||||||
|
ByteArrayOutputStream out = new ByteArrayOutputStream(Math.max(64, gz.length * 4))) {
|
||||||
|
in.transferTo(out);
|
||||||
|
return out.toByteArray();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int indexOfNewline(byte[] bytes, int from) {
|
||||||
|
for (int i = Math.max(0, from); i < bytes.length; i++) {
|
||||||
|
if (bytes[i] == '\n') {
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int nvl(Integer value) {
|
||||||
|
return value == null ? 0 : value;
|
||||||
|
}
|
||||||
|
}
|
||||||
+177
@@ -0,0 +1,177 @@
|
|||||||
|
package com.nanri.aiimage.modules.devicelog.storage;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.config.DeviceLogOssProperties;
|
||||||
|
import io.minio.BucketExistsArgs;
|
||||||
|
import io.minio.GetObjectArgs;
|
||||||
|
import io.minio.GetObjectResponse;
|
||||||
|
import io.minio.ListObjectsArgs;
|
||||||
|
import io.minio.MakeBucketArgs;
|
||||||
|
import io.minio.MinioClient;
|
||||||
|
import io.minio.PutObjectArgs;
|
||||||
|
import io.minio.RemoveObjectsArgs;
|
||||||
|
import io.minio.Result;
|
||||||
|
import io.minio.messages.DeleteError;
|
||||||
|
import io.minio.messages.DeleteObject;
|
||||||
|
import io.minio.messages.Item;
|
||||||
|
import jakarta.annotation.PostConstruct;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.io.ByteArrayInputStream;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 设备日志的独立对象存储客户端(主机B 自建 MinIO,非业务 OSS)。
|
||||||
|
*
|
||||||
|
* <p>只存 gzip 片段原文,不做重压缩;桶的 7 天过期规则在部署时由运维用 mc 配置,
|
||||||
|
* 本类不负责生命周期管理。
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class DeviceLogStorageService {
|
||||||
|
|
||||||
|
private final DeviceLogOssProperties properties;
|
||||||
|
|
||||||
|
private volatile MinioClient client;
|
||||||
|
|
||||||
|
@PostConstruct
|
||||||
|
void init() {
|
||||||
|
if (!properties.configured()) {
|
||||||
|
log.warn("[device-log] aiimage.device-log-oss 未配置(endpoint/凭据/桶),"
|
||||||
|
+ "日志上报与管理接口将不可用;生产必须通过 AIIMAGE_DEVICE_LOG_OSS_* 环境变量注入");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
MinioClient built = MinioClient.builder()
|
||||||
|
.endpoint(properties.getEndpoint())
|
||||||
|
.credentials(properties.getAccessKeyId(), properties.getAccessKeySecret())
|
||||||
|
.build();
|
||||||
|
// 收紧超时:日志接口不能被慢存储拖挂(默认读超时 5 分钟)
|
||||||
|
built.setTimeout(10_000, 60_000, 60_000);
|
||||||
|
boolean exists = built.bucketExists(BucketExistsArgs.builder()
|
||||||
|
.bucket(properties.getBucket()).build());
|
||||||
|
if (!exists) {
|
||||||
|
built.makeBucket(MakeBucketArgs.builder().bucket(properties.getBucket()).build());
|
||||||
|
log.info("[device-log] 已创建日志桶 bucket={}", properties.getBucket());
|
||||||
|
}
|
||||||
|
this.client = built;
|
||||||
|
log.info("[device-log] 日志对象存储已就绪 endpoint={} bucket={} 保留天数={}",
|
||||||
|
properties.getEndpoint(), properties.getBucket(), properties.retentionDaysOrDefault());
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.error("[device-log] 日志对象存储初始化失败 endpoint={} bucket={},日志功能不可用: {}",
|
||||||
|
properties.getEndpoint(), properties.getBucket(), ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean enabled() {
|
||||||
|
return client != null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String bucket() {
|
||||||
|
return properties.getBucket();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 写入一个 gzip 片段(同 key 覆盖写,幂等)。 */
|
||||||
|
public void putPart(String objectKey, byte[] gzipBytes) {
|
||||||
|
MinioClient c = requireClient();
|
||||||
|
try (ByteArrayInputStream in = new ByteArrayInputStream(gzipBytes)) {
|
||||||
|
c.putObject(PutObjectArgs.builder()
|
||||||
|
.bucket(bucket())
|
||||||
|
.object(objectKey)
|
||||||
|
.stream(in, gzipBytes.length, -1)
|
||||||
|
.contentType("application/gzip")
|
||||||
|
.build());
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException("写日志对象失败 key=" + objectKey + " err=" + ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 列出前缀下全部对象 key(按 key 升序;key 内的 offset 为零填充,字典序即偏移序)。 */
|
||||||
|
public List<String> listParts(String prefix) {
|
||||||
|
MinioClient c = requireClient();
|
||||||
|
List<String> keys = new ArrayList<>();
|
||||||
|
try {
|
||||||
|
Iterable<Result<Item>> results = c.listObjects(ListObjectsArgs.builder()
|
||||||
|
.bucket(bucket())
|
||||||
|
.prefix(prefix)
|
||||||
|
.recursive(true)
|
||||||
|
.build());
|
||||||
|
for (Result<Item> result : results) {
|
||||||
|
keys.add(result.get().objectName());
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException("列日志对象失败 prefix=" + prefix + " err=" + ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
keys.sort(String::compareTo);
|
||||||
|
return keys;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 读取一个 gzip 片段的原始字节(未解压)。 */
|
||||||
|
public byte[] readPartBytes(String objectKey) {
|
||||||
|
MinioClient c = requireClient();
|
||||||
|
try (GetObjectResponse response = c.getObject(GetObjectArgs.builder()
|
||||||
|
.bucket(bucket()).object(objectKey).build())) {
|
||||||
|
return response.readAllBytes();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException("读日志对象失败 key=" + objectKey + " err=" + ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 打开一个 gzip 片段的流(调用方负责关闭;下载拼接时避免整段进内存)。 */
|
||||||
|
public InputStream openPartStream(String objectKey) {
|
||||||
|
MinioClient c = requireClient();
|
||||||
|
try {
|
||||||
|
return c.getObject(GetObjectArgs.builder()
|
||||||
|
.bucket(bucket()).object(objectKey).build());
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException("打开日志对象流失败 key=" + objectKey + " err=" + ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 批量删除对象;返回删除失败的 key 列表。 */
|
||||||
|
public List<String> deleteParts(List<String> objectKeys) {
|
||||||
|
if (objectKeys == null || objectKeys.isEmpty()) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
MinioClient c = requireClient();
|
||||||
|
List<DeleteObject> targets = objectKeys.stream().map(DeleteObject::new).toList();
|
||||||
|
List<String> failed = new ArrayList<>();
|
||||||
|
try {
|
||||||
|
Iterable<Result<DeleteError>> results = c.removeObjects(RemoveObjectsArgs.builder()
|
||||||
|
.bucket(bucket()).objects(targets).build());
|
||||||
|
for (Result<DeleteError> result : results) {
|
||||||
|
DeleteError error = result.get();
|
||||||
|
failed.add(error.objectName());
|
||||||
|
log.warn("[device-log] 删除对象失败 key={} err={}", error.objectName(), error.message());
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException("批量删除日志对象失败 err=" + ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
return failed;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 探测连通性(上传接口的错误提示用)。 */
|
||||||
|
public boolean ping() {
|
||||||
|
if (client == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return client.bucketExists(BucketExistsArgs.builder().bucket(bucket()).build());
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[device-log] 存储连通性探测失败: {}", ex.getMessage());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private MinioClient requireClient() {
|
||||||
|
MinioClient c = client;
|
||||||
|
if (c == null) {
|
||||||
|
throw new IllegalStateException("日志对象存储未配置或初始化失败");
|
||||||
|
}
|
||||||
|
return c;
|
||||||
|
}
|
||||||
|
}
|
||||||
+34
-7
@@ -11,6 +11,8 @@ import lombok.RequiredArgsConstructor;
|
|||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
import org.springframework.transaction.support.TransactionSynchronization;
|
||||||
|
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||||
import org.springframework.web.multipart.MultipartFile;
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
@@ -187,6 +189,13 @@ public class DigitalHumanVersionService {
|
|||||||
return toVo(entity);
|
return toVo(entity);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 删除版本。
|
||||||
|
*
|
||||||
|
* <p>事务边界:MinIO 对象删除移到事务提交后执行 —— 单次删除超时 120s、重试 3 次,
|
||||||
|
* 放在 DB 事务里会长时间占用连接(项目在 TaskScopePayloadStorageService 已写明该约束)。
|
||||||
|
* 顺序仍是「先删库、后删对象」,与改前一致。
|
||||||
|
*/
|
||||||
@Transactional
|
@Transactional
|
||||||
public void deleteVersion(String version) {
|
public void deleteVersion(String version) {
|
||||||
DigitalHumanVersionEntity entity = versionMapper.selectOne(new LambdaQueryWrapper<DigitalHumanVersionEntity>()
|
DigitalHumanVersionEntity entity = versionMapper.selectOne(new LambdaQueryWrapper<DigitalHumanVersionEntity>()
|
||||||
@@ -198,15 +207,33 @@ public class DigitalHumanVersionService {
|
|||||||
throw new BusinessException("最新版本不能删除");
|
throw new BusinessException("最新版本不能删除");
|
||||||
}
|
}
|
||||||
|
|
||||||
// 删除 MinIO 文件
|
|
||||||
try {
|
|
||||||
ossStorageService.deleteObject(entity.getOssObjectKey());
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.warn("删除 MinIO 文件失败:{}", entity.getOssObjectKey(), e);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 删除数据库记录
|
// 删除数据库记录
|
||||||
versionMapper.deleteById(entity.getId());
|
versionMapper.deleteById(entity.getId());
|
||||||
|
|
||||||
|
// 删除 MinIO 文件(事务提交后)
|
||||||
|
deleteObjectAfterCommit(entity.getOssObjectKey());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 事务提交后删 MinIO 对象;无事务时立即执行。 */
|
||||||
|
private void deleteObjectAfterCommit(String objectKey) {
|
||||||
|
if (TransactionSynchronizationManager.isSynchronizationActive()) {
|
||||||
|
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
|
||||||
|
@Override
|
||||||
|
public void afterCommit() {
|
||||||
|
deleteObjectQuietly(objectKey);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
deleteObjectQuietly(objectKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void deleteObjectQuietly(String objectKey) {
|
||||||
|
try {
|
||||||
|
ossStorageService.deleteObject(objectKey);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("删除 MinIO 文件失败:{}", objectKey, e);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getDownloadUrl(String version) {
|
public String getDownloadUrl(String version) {
|
||||||
|
|||||||
+159
-1
@@ -5,10 +5,14 @@ import io.micrometer.core.instrument.DistributionSummary;
|
|||||||
import io.micrometer.core.instrument.MeterRegistry;
|
import io.micrometer.core.instrument.MeterRegistry;
|
||||||
import io.micrometer.core.instrument.Timer;
|
import io.micrometer.core.instrument.Timer;
|
||||||
import io.minio.GetObjectArgs;
|
import io.minio.GetObjectArgs;
|
||||||
|
import io.minio.ListObjectsArgs;
|
||||||
import io.minio.MinioClient;
|
import io.minio.MinioClient;
|
||||||
import io.minio.PutObjectArgs;
|
import io.minio.PutObjectArgs;
|
||||||
import io.minio.RemoveObjectArgs;
|
import io.minio.RemoveObjectArgs;
|
||||||
|
import io.minio.Result;
|
||||||
import io.minio.StatObjectArgs;
|
import io.minio.StatObjectArgs;
|
||||||
|
import io.minio.errors.ErrorResponseException;
|
||||||
|
import io.minio.messages.Item;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import okhttp3.ConnectionPool;
|
import okhttp3.ConnectionPool;
|
||||||
import okhttp3.Dispatcher;
|
import okhttp3.Dispatcher;
|
||||||
@@ -20,7 +24,11 @@ import org.springframework.stereotype.Service;
|
|||||||
import java.io.ByteArrayInputStream;
|
import java.io.ByteArrayInputStream;
|
||||||
import java.io.ByteArrayOutputStream;
|
import java.io.ByteArrayOutputStream;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.UUID;
|
||||||
import java.util.concurrent.Semaphore;
|
import java.util.concurrent.Semaphore;
|
||||||
import java.util.concurrent.ThreadLocalRandom;
|
import java.util.concurrent.ThreadLocalRandom;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
@@ -38,6 +46,17 @@ public class RustfsObjectStorageService {
|
|||||||
private static final String OP_STAT = "stat";
|
private static final String OP_STAT = "stat";
|
||||||
private static final String OP_TOTAL = "total";
|
private static final String OP_TOTAL = "total";
|
||||||
|
|
||||||
|
/** S3 确定性错误码:重试不会改变结果,应立即失败而不是白打两次请求。 */
|
||||||
|
private static final Set<String> NON_RETRYABLE_S3_CODES = Set.of(
|
||||||
|
"NoSuchKey", "NoSuchBucket", "NoSuchVersion",
|
||||||
|
"AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "InvalidBucketName");
|
||||||
|
|
||||||
|
/** 标准 UUID 字符串长度(8-4-4-4-12),用于识别版本化对象 key。 */
|
||||||
|
private static final int UUID_STRING_LENGTH = 36;
|
||||||
|
|
||||||
|
/** 兄弟对象兜底列出的上限:只为找回同槽位的版本化对象,不需要列全。 */
|
||||||
|
private static final int MAX_SIBLING_LIST_KEYS = 50;
|
||||||
|
|
||||||
private final TransientStorageProperties properties;
|
private final TransientStorageProperties properties;
|
||||||
private final ObjectProvider<MeterRegistry> meterRegistryProvider;
|
private final ObjectProvider<MeterRegistry> meterRegistryProvider;
|
||||||
private final ObjectProvider<RustfsDeleteRetryService> deleteRetryServiceProvider;
|
private final ObjectProvider<RustfsDeleteRetryService> deleteRetryServiceProvider;
|
||||||
@@ -99,7 +118,22 @@ public class RustfsObjectStorageService {
|
|||||||
return uploadBytes(objectKey, bytes, verifyAfterUpload);
|
return uploadBytes(objectKey, bytes, verifyAfterUpload);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 三参重载:是否做「上传失败补偿删除」按对象 key 形态自动判定。
|
||||||
|
*
|
||||||
|
* <p>只有版本化 key(末段以 UUID 结尾)是本次写入独占的;确定性 key 会被重传重写复用,
|
||||||
|
* 删它就可能删掉别的 DB 行仍在引用的对象(2026-09-17 线上任务 28459 的载荷对象就是这么丢的)。
|
||||||
|
*/
|
||||||
public String uploadBytes(String objectKey, byte[] content, boolean verifyAfterUpload) {
|
public String uploadBytes(String objectKey, byte[] content, boolean verifyAfterUpload) {
|
||||||
|
return uploadBytes(objectKey, content, verifyAfterUpload, isVersionedObjectKey(objectKey));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param compensateDeleteOnFailure put 已完成、但后续可见性校验失败时,是否把该对象排进删除补偿队列。
|
||||||
|
* 仅当调用方能确认「该对象不会被其它写入复用时」才可传 true。
|
||||||
|
*/
|
||||||
|
public String uploadBytes(String objectKey, byte[] content, boolean verifyAfterUpload,
|
||||||
|
boolean compensateDeleteOnFailure) {
|
||||||
long deadlineNanos = operationDeadlineNanos();
|
long deadlineNanos = operationDeadlineNanos();
|
||||||
if (!isConfigured()) {
|
if (!isConfigured()) {
|
||||||
throw new IllegalStateException("transient storage is not configured");
|
throw new IllegalStateException("transient storage is not configured");
|
||||||
@@ -129,13 +163,45 @@ public class RustfsObjectStorageService {
|
|||||||
}
|
}
|
||||||
return uploadedObjectKey;
|
return uploadedObjectKey;
|
||||||
} catch (RuntimeException ex) {
|
} catch (RuntimeException ex) {
|
||||||
if (putCompleted.get()) {
|
if (putCompleted.get() && compensateDeleteOnFailure) {
|
||||||
enqueueDeleteRetry(objectKey, ex);
|
enqueueDeleteRetry(objectKey, ex);
|
||||||
|
} else if (putCompleted.get()) {
|
||||||
|
// 共享 key 会被重传重写:此处删除可能删掉别的行正在引用的对象,交给保留期清理兜底。
|
||||||
|
// 线上任务 28459 的 chunk-462/473/484 就是被这条无条件删除队列删掉的。
|
||||||
|
log.warn("[rustfs] 跳过上传失败补偿删除(对象非本次独占,可能被复用)objectKey={} err={}",
|
||||||
|
objectKey, ex.getMessage());
|
||||||
}
|
}
|
||||||
throw ex;
|
throw ex;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 对象 key 是否为「本次写入独占」的版本化 key:末段(去掉 {@code .json} 后缀)以 UUID 结尾。
|
||||||
|
*
|
||||||
|
* <p>只看末段——UUID 出现在中间段(如 scopeHash)不代表该对象被独占;解析失败一律按共享处理
|
||||||
|
* (保守:宁可留下孤儿对象,也不删掉可能仍被引用的对象)。
|
||||||
|
*/
|
||||||
|
static boolean isVersionedObjectKey(String objectKey) {
|
||||||
|
if (objectKey == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
String key = objectKey.trim();
|
||||||
|
if (key.endsWith(".json")) {
|
||||||
|
key = key.substring(0, key.length() - ".json".length());
|
||||||
|
}
|
||||||
|
int slash = key.lastIndexOf('/');
|
||||||
|
String lastSegment = slash < 0 ? key : key.substring(slash + 1);
|
||||||
|
if (lastSegment.length() < UUID_STRING_LENGTH) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
UUID.fromString(lastSegment.substring(lastSegment.length() - UUID_STRING_LENGTH));
|
||||||
|
return true;
|
||||||
|
} catch (IllegalArgumentException ex) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public String readObjectAsString(String objectKey) {
|
public String readObjectAsString(String objectKey) {
|
||||||
byte[] bytes = readObjectBytes(objectKey);
|
byte[] bytes = readObjectBytes(objectKey);
|
||||||
return new String(bytes, StandardCharsets.UTF_8);
|
return new String(bytes, StandardCharsets.UTF_8);
|
||||||
@@ -175,6 +241,43 @@ public class RustfsObjectStorageService {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 列出前缀下的对象 key,按最后修改时间倒序(最新在前)。
|
||||||
|
*
|
||||||
|
* <p>只服务于「指针指向的对象已不存在、需要找回同槽位的版本化兄弟对象」这一兜底路径,
|
||||||
|
* 因此刻意不做重试、不参与失败窗口记账:列出失败直接抛错,由调用方按原错误语义处理。
|
||||||
|
*/
|
||||||
|
public List<String> listObjectKeysNewestFirst(String prefix, int limit) {
|
||||||
|
if (!isConfigured()) {
|
||||||
|
throw new IllegalStateException("transient storage is not configured");
|
||||||
|
}
|
||||||
|
int safeLimit = Math.max(1, Math.min(limit, MAX_SIBLING_LIST_KEYS));
|
||||||
|
long deadlineNanos = operationDeadlineNanos();
|
||||||
|
List<String[]> entries = new ArrayList<>();
|
||||||
|
try {
|
||||||
|
Iterable<Result<Item>> results = buildClient(deadlineNanos).listObjects(ListObjectsArgs.builder()
|
||||||
|
.bucket(properties.getBucket())
|
||||||
|
.prefix(prefix == null ? "" : prefix)
|
||||||
|
.recursive(true)
|
||||||
|
.maxKeys(safeLimit)
|
||||||
|
.build());
|
||||||
|
for (Result<Item> result : results) {
|
||||||
|
Item item = result.get();
|
||||||
|
entries.add(new String[]{item.objectName(),
|
||||||
|
item.lastModified() == null ? "" : item.lastModified().toString()});
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException("transient storage list failed prefix=" + prefix
|
||||||
|
+ " err=" + ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
entries.sort((left, right) -> right[1].compareTo(left[1]));
|
||||||
|
List<String> keys = new ArrayList<>(entries.size());
|
||||||
|
for (String[] entry : entries) {
|
||||||
|
keys.add(entry[0]);
|
||||||
|
}
|
||||||
|
return keys;
|
||||||
|
}
|
||||||
|
|
||||||
public void deleteObject(String objectKey) {
|
public void deleteObject(String objectKey) {
|
||||||
deleteObject(objectKey, true, operationDeadlineNanos());
|
deleteObject(objectKey, true, operationDeadlineNanos());
|
||||||
}
|
}
|
||||||
@@ -238,6 +341,12 @@ public class RustfsObjectStorageService {
|
|||||||
resetFailureWindow(operation);
|
resetFailureWindow(operation);
|
||||||
}
|
}
|
||||||
recordOperation(operation, "success", elapsedNanos(startedAt));
|
recordOperation(operation, "success", elapsedNanos(startedAt));
|
||||||
|
if (attempt > 1) {
|
||||||
|
// 重试后成功必须留 INFO 结论:线上一天上千条 "operation failed, retrying"
|
||||||
|
// 却没有任何结论日志,无法判断这些上传最后到底落盘了没有。
|
||||||
|
log.info("[rustfs] 重试后成功 operation={} objectKey={} attempt={}/{} durationMs={} bytes={}",
|
||||||
|
operation, objectKey, attempt, maxRetries, elapsedMillis(startedAt), bytes);
|
||||||
|
}
|
||||||
log.debug("[rustfs] operation success operation={} objectKey={} attempt={}/{} durationMs={} bytes={}",
|
log.debug("[rustfs] operation success operation={} objectKey={} attempt={}/{} durationMs={} bytes={}",
|
||||||
operation, objectKey, attempt, maxRetries, elapsedMillis(startedAt), bytes);
|
operation, objectKey, attempt, maxRetries, elapsedMillis(startedAt), bytes);
|
||||||
return result;
|
return result;
|
||||||
@@ -245,6 +354,15 @@ public class RustfsObjectStorageService {
|
|||||||
last = ex;
|
last = ex;
|
||||||
recordOperation(operation, attempt < maxRetries ? "retry" : "failure", elapsedNanos(startedAt));
|
recordOperation(operation, attempt < maxRetries ? "retry" : "failure", elapsedNanos(startedAt));
|
||||||
recordFailure(operation, objectKey, ex);
|
recordFailure(operation, objectKey, ex);
|
||||||
|
if (isNonRetryable(ex)) {
|
||||||
|
// 确定性错误(NoSuchKey / AccessDenied…):重试多少次结果都一样。
|
||||||
|
// 线上 read 一个已被清理的 chunk 就会连打 3 次请求、还被记成 ERROR。
|
||||||
|
log.warn("[rustfs] 确定性错误,不重试 operation={} objectKey={} err={}",
|
||||||
|
operation, objectKey, ex.getMessage());
|
||||||
|
throw ex instanceof RuntimeException runtimeException
|
||||||
|
? runtimeException
|
||||||
|
: new IllegalStateException(ex);
|
||||||
|
}
|
||||||
if (attempt < maxRetries) {
|
if (attempt < maxRetries) {
|
||||||
delayMillis = retryDelayMillis(attempt);
|
delayMillis = retryDelayMillis(attempt);
|
||||||
log.warn("[rustfs] operation failed, retrying operation={} objectKey={} attempt={}/{} delayMs={} err={}",
|
log.warn("[rustfs] operation failed, retrying operation={} objectKey={} attempt={}/{} delayMs={} err={}",
|
||||||
@@ -262,6 +380,9 @@ public class RustfsObjectStorageService {
|
|||||||
operation, objectKey, deadlineNanos);
|
operation, objectKey, deadlineNanos);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// 重试耗尽:留 ERROR 结论 + 最后一次错误,否则只看到一串 retrying,无从判断是否真丢数据
|
||||||
|
log.error("[rustfs] 重试耗尽,最终失败 operation={} objectKey={} maxRetries={} 最后一次错误={}",
|
||||||
|
operation, objectKey, maxRetries, last == null ? "无" : last.getMessage(), last);
|
||||||
throw new IllegalStateException("failed to " + operation + " payload in transient storage", last);
|
throw new IllegalStateException("failed to " + operation + " payload in transient storage", last);
|
||||||
} finally {
|
} finally {
|
||||||
if (totalAcquired) {
|
if (totalAcquired) {
|
||||||
@@ -270,6 +391,27 @@ public class RustfsObjectStorageService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 是否为「重试也没用」的确定性错误(NoSuchKey / AccessDenied / 签名错误…)。
|
||||||
|
*
|
||||||
|
* <p>只有网络类与 5xx 类失败才值得重试;确定性错误重试多少次结果都一样。
|
||||||
|
*/
|
||||||
|
private static boolean isNonRetryable(Throwable error) {
|
||||||
|
Throwable cursor = error;
|
||||||
|
while (cursor != null) {
|
||||||
|
if (cursor instanceof ErrorResponseException responseException) {
|
||||||
|
String code = responseException.errorResponse() == null
|
||||||
|
? null
|
||||||
|
: responseException.errorResponse().code();
|
||||||
|
if (code != null && NON_RETRYABLE_S3_CODES.contains(code)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cursor = cursor.getCause();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
private void acquirePermit(String operation, String objectKey, Semaphore semaphore, long deadlineNanos) {
|
private void acquirePermit(String operation, String objectKey, Semaphore semaphore, long deadlineNanos) {
|
||||||
try {
|
try {
|
||||||
checkDeadline(operation, objectKey, deadlineNanos);
|
checkDeadline(operation, objectKey, deadlineNanos);
|
||||||
@@ -336,6 +478,22 @@ public class RustfsObjectStorageService {
|
|||||||
Math.max(1L, properties.getConnectionPoolKeepAliveMillis()),
|
Math.max(1L, properties.getConnectionPoolKeepAliveMillis()),
|
||||||
TimeUnit.MILLISECONDS))
|
TimeUnit.MILLISECONDS))
|
||||||
.retryOnConnectionFailure(true)
|
.retryOnConnectionFailure(true)
|
||||||
|
.addNetworkInterceptor(chain -> {
|
||||||
|
okhttp3.Request request = chain.request();
|
||||||
|
okhttp3.RequestBody body = request.body();
|
||||||
|
// OkHttp 对「长度为 0 的请求体」不会写 Content-Length,于是请求既无
|
||||||
|
// Content-Length 也无 Transfer-Encoding(HTTP/1.1 不允许这样)。
|
||||||
|
// RustFS 对此直接回 411 Length Required,客户端读到不完整响应就报
|
||||||
|
// unexpected end of stream,进而重试——线上抓包实测 90 秒内 515 次 411,
|
||||||
|
// 而上传空内容(content 为 null/空)在业务里是常态。
|
||||||
|
// 用 network interceptor 在协议层补上该头(普通 interceptor 会被
|
||||||
|
// BridgeInterceptor 按 body 长度覆盖掉,加了也不生效)。
|
||||||
|
if (body != null && body.contentLength() == 0L
|
||||||
|
&& request.header("Content-Length") == null) {
|
||||||
|
request = request.newBuilder().header("Content-Length", "0").build();
|
||||||
|
}
|
||||||
|
return chain.proceed(request);
|
||||||
|
})
|
||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
return httpClient;
|
return httpClient;
|
||||||
|
|||||||
+9
-1
@@ -40,6 +40,8 @@ public class ImageVideoAsyncTaskService {
|
|||||||
private static final int DISPATCH_BATCH_SIZE = 20;
|
private static final int DISPATCH_BATCH_SIZE = 20;
|
||||||
private static final int POLL_BATCH_SIZE = 50;
|
private static final int POLL_BATCH_SIZE = 50;
|
||||||
private static final int FAILED_TASK_RETENTION_MINUTES = 10;
|
private static final int FAILED_TASK_RETENTION_MINUTES = 10;
|
||||||
|
/** 过期失败任务单轮删除上限:一条无 LIMIT 的 DELETE 会长时间持锁,改成分批(每轮扫描都会再清)。 */
|
||||||
|
private static final int EXPIRED_DELETE_BATCH_SIZE = 500;
|
||||||
private static final Set<String> TERMINAL_STATUSES = Set.of(
|
private static final Set<String> TERMINAL_STATUSES = Set.of(
|
||||||
"SUCCESS", "SUCCEEDED", "COMPLETED", "DONE", "FINISHED",
|
"SUCCESS", "SUCCEEDED", "COMPLETED", "DONE", "FINISHED",
|
||||||
"FAILED", "FAIL", "ERROR", "CANCELED", "CANCELLED"
|
"FAILED", "FAIL", "ERROR", "CANCELED", "CANCELLED"
|
||||||
@@ -142,7 +144,10 @@ public class ImageVideoAsyncTaskService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try (jobLock) {
|
try (jobLock) {
|
||||||
|
// 只取主键:本表含 5 个 LONGTEXT 列(请求/响应正文),而这里只用来发起
|
||||||
|
// executeTask(id)。每秒扫一轮还拉全部大字段属于纯浪费(2026-09-15 优化)。
|
||||||
List<ImageVideoAsyncTaskEntity> tasks = taskMapper.selectList(new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>()
|
List<ImageVideoAsyncTaskEntity> tasks = taskMapper.selectList(new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>()
|
||||||
|
.select(ImageVideoAsyncTaskEntity::getId)
|
||||||
.eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.PENDING.name())
|
.eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.PENDING.name())
|
||||||
.and(q -> q.eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, currentInstanceId())
|
.and(q -> q.eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, currentInstanceId())
|
||||||
.or().isNull(ImageVideoAsyncTaskEntity::getOwnerInstanceId)
|
.or().isNull(ImageVideoAsyncTaskEntity::getOwnerInstanceId)
|
||||||
@@ -161,7 +166,9 @@ public class ImageVideoAsyncTaskService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try (jobLock) {
|
try (jobLock) {
|
||||||
|
// 同 dispatchPendingTasks:只取主键,避免每 5 秒把 LONGTEXT 正文整列拉回
|
||||||
List<ImageVideoAsyncTaskEntity> tasks = taskMapper.selectList(new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>()
|
List<ImageVideoAsyncTaskEntity> tasks = taskMapper.selectList(new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>()
|
||||||
|
.select(ImageVideoAsyncTaskEntity::getId)
|
||||||
.eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.WAITING.name())
|
.eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.WAITING.name())
|
||||||
.and(q -> q.eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, currentInstanceId())
|
.and(q -> q.eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, currentInstanceId())
|
||||||
.or().isNull(ImageVideoAsyncTaskEntity::getOwnerInstanceId)
|
.or().isNull(ImageVideoAsyncTaskEntity::getOwnerInstanceId)
|
||||||
@@ -234,7 +241,8 @@ public class ImageVideoAsyncTaskService {
|
|||||||
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(FAILED_TASK_RETENTION_MINUTES);
|
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(FAILED_TASK_RETENTION_MINUTES);
|
||||||
int deleted = taskMapper.delete(new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>()
|
int deleted = taskMapper.delete(new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>()
|
||||||
.eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.FAILED.name())
|
.eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.FAILED.name())
|
||||||
.lt(ImageVideoAsyncTaskEntity::getUpdatedAt, cutoff));
|
.lt(ImageVideoAsyncTaskEntity::getUpdatedAt, cutoff)
|
||||||
|
.last("LIMIT " + EXPIRED_DELETE_BATCH_SIZE));
|
||||||
if (deleted > 0) {
|
if (deleted > 0) {
|
||||||
log.info("[image-video] removed expired failed tasks count={}", deleted);
|
log.info("[image-video] removed expired failed tasks count={}", deleted);
|
||||||
}
|
}
|
||||||
|
|||||||
+9
@@ -1,6 +1,8 @@
|
|||||||
package com.nanri.aiimage.modules.invalidasin.model.entity;
|
package com.nanri.aiimage.modules.invalidasin.model.entity;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.annotation.FieldStrategy;
|
||||||
import com.baomidou.mybatisplus.annotation.IdType;
|
import com.baomidou.mybatisplus.annotation.IdType;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableField;
|
||||||
import com.baomidou.mybatisplus.annotation.TableId;
|
import com.baomidou.mybatisplus.annotation.TableId;
|
||||||
import com.baomidou.mybatisplus.annotation.TableName;
|
import com.baomidou.mybatisplus.annotation.TableName;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -18,5 +20,12 @@ public class InvalidAsinDataEntity {
|
|||||||
private Long groupId;
|
private Long groupId;
|
||||||
private String recordSource;
|
private String recordSource;
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
|
/**
|
||||||
|
* 更新时间:由数据库维护(DEFAULT CURRENT_TIMESTAMP + ON UPDATE CURRENT_TIMESTAMP)。
|
||||||
|
*
|
||||||
|
* <p>禁止应用显式写:MySQL 在 UPDATE 语句显式给该列赋值时不会触发自动更新,
|
||||||
|
* 而本表写回为 selectById → 改字段 → updateById(实体带着旧值),一旦写回就会冻结更新时间。
|
||||||
|
*/
|
||||||
|
@TableField(value = "updated_at", insertStrategy = FieldStrategy.NEVER, updateStrategy = FieldStrategy.NEVER)
|
||||||
private LocalDateTime updatedAt;
|
private LocalDateTime updatedAt;
|
||||||
}
|
}
|
||||||
|
|||||||
+3
@@ -29,4 +29,7 @@ public class InvalidAsinDataItemVo {
|
|||||||
|
|
||||||
@Schema(description = "创建时间")
|
@Schema(description = "创建时间")
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
|
|
||||||
|
@Schema(description = "更新时间")
|
||||||
|
private LocalDateTime updatedAt;
|
||||||
}
|
}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user